




































































































































































































































    1 





























    1 





    1 
    1 


















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
// SPDX-License-Identifier: GPL-2.0-only
//
// ethtool interface for Ethernet PSE (Power Sourcing Equipment)
// and PD (Powered Device)
//
// Copyright (c) 2022 Pengutronix, Oleksij Rempel <kernel@pengutronix.de>
//

#include "common.h"
#include "linux/pse-pd/pse.h"
#include "netlink.h"
#include <linux/ethtool_netlink.h>
#include <linux/ethtool.h>
#include <linux/phy.h>

struct pse_req_info {
        struct ethnl_req_info base;
};

struct pse_reply_data {
        struct ethnl_reply_data        base;
        struct ethtool_pse_control_status status;
};

#define PSE_REPDATA(__reply_base) \
        container_of(__reply_base, struct pse_reply_data, base)

/* PSE_GET */

const struct nla_policy ethnl_pse_get_policy[ETHTOOL_A_PSE_HEADER + 1] = {
        [ETHTOOL_A_PSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy_phy),
};

static int pse_get_pse_attributes(struct phy_device *phydev,
                                  struct netlink_ext_ack *extack,
                                  struct pse_reply_data *data)
{
        if (!phydev) {
                NL_SET_ERR_MSG(extack, "No PHY found");
                return -EOPNOTSUPP;
        }

        if (!phydev->psec) {
                NL_SET_ERR_MSG(extack, "No PSE is attached");
                return -EOPNOTSUPP;
        }

        memset(&data->status, 0, sizeof(data->status));

        return pse_ethtool_get_status(phydev->psec, extack, &data->status);
}

static int pse_prepare_data(const struct ethnl_req_info *req_base,
                            struct ethnl_reply_data *reply_base,
                            const struct genl_info *info)
{
        struct pse_reply_data *data = PSE_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        int ret;

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                return ret;

        phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_PSE_HEADER,
                                      info->extack);
        if (IS_ERR(phydev))
                return -ENODEV;

        ret = pse_get_pse_attributes(phydev, info->extack, data);

        ethnl_ops_complete(dev);

        return ret;
}

static int pse_reply_size(const struct ethnl_req_info *req_base,
                          const struct ethnl_reply_data *reply_base)
{
        const struct pse_reply_data *data = PSE_REPDATA(reply_base);
        const struct ethtool_pse_control_status *st = &data->status;
        int len = 0;

        if (st->podl_admin_state > 0)
                len += nla_total_size(sizeof(u32)); /* _PODL_PSE_ADMIN_STATE */
        if (st->podl_pw_status > 0)
                len += nla_total_size(sizeof(u32)); /* _PODL_PSE_PW_D_STATUS */
        if (st->c33_admin_state > 0)
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_ADMIN_STATE */
        if (st->c33_pw_status > 0)
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_PW_D_STATUS */
        if (st->c33_pw_class > 0)
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_PW_CLASS */
        if (st->c33_actual_pw > 0)
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_ACTUAL_PW */
        if (st->c33_ext_state_info.c33_pse_ext_state > 0) {
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_EXT_STATE */
                if (st->c33_ext_state_info.__c33_pse_ext_substate > 0)
                        /* _C33_PSE_EXT_SUBSTATE */
                        len += nla_total_size(sizeof(u32));
        }
        if (st->c33_avail_pw_limit > 0)
                /* _C33_AVAIL_PSE_PW_LIMIT */
                len += nla_total_size(sizeof(u32));
        if (st->c33_pw_limit_nb_ranges > 0)
                /* _C33_PSE_PW_LIMIT_RANGES */
                len += st->c33_pw_limit_nb_ranges *
                       (nla_total_size(0) +
                        nla_total_size(sizeof(u32)) * 2);

        return len;
}

static int pse_put_pw_limit_ranges(struct sk_buff *skb,
                                   const struct ethtool_pse_control_status *st)
{
        const struct ethtool_c33_pse_pw_limit_range *pw_limit_ranges;
        int i;

        pw_limit_ranges = st->c33_pw_limit_ranges;
        for (i = 0; i < st->c33_pw_limit_nb_ranges; i++) {
                struct nlattr *nest;

                nest = nla_nest_start(skb, ETHTOOL_A_C33_PSE_PW_LIMIT_RANGES);
                if (!nest)
                        return -EMSGSIZE;

                if (nla_put_u32(skb, ETHTOOL_A_C33_PSE_PW_LIMIT_MIN,
                                pw_limit_ranges->min) ||
                    nla_put_u32(skb, ETHTOOL_A_C33_PSE_PW_LIMIT_MAX,
                                pw_limit_ranges->max)) {
                        nla_nest_cancel(skb, nest);
                        return -EMSGSIZE;
                }
                nla_nest_end(skb, nest);
                pw_limit_ranges++;
        }

        return 0;
}

static int pse_fill_reply(struct sk_buff *skb,
                          const struct ethnl_req_info *req_base,
                          const struct ethnl_reply_data *reply_base)
{
        const struct pse_reply_data *data = PSE_REPDATA(reply_base);
        const struct ethtool_pse_control_status *st = &data->status;

        if (st->podl_admin_state > 0 &&
            nla_put_u32(skb, ETHTOOL_A_PODL_PSE_ADMIN_STATE,
                        st->podl_admin_state))
                return -EMSGSIZE;

        if (st->podl_pw_status > 0 &&
            nla_put_u32(skb, ETHTOOL_A_PODL_PSE_PW_D_STATUS,
                        st->podl_pw_status))
                return -EMSGSIZE;

        if (st->c33_admin_state > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_ADMIN_STATE,
                        st->c33_admin_state))
                return -EMSGSIZE;

        if (st->c33_pw_status > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_PW_D_STATUS,
                        st->c33_pw_status))
                return -EMSGSIZE;

        if (st->c33_pw_class > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_PW_CLASS,
                        st->c33_pw_class))
                return -EMSGSIZE;

        if (st->c33_actual_pw > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_ACTUAL_PW,
                        st->c33_actual_pw))
                return -EMSGSIZE;

        if (st->c33_ext_state_info.c33_pse_ext_state > 0) {
                if (nla_put_u32(skb, ETHTOOL_A_C33_PSE_EXT_STATE,
                                st->c33_ext_state_info.c33_pse_ext_state))
                        return -EMSGSIZE;

                if (st->c33_ext_state_info.__c33_pse_ext_substate > 0 &&
                    nla_put_u32(skb, ETHTOOL_A_C33_PSE_EXT_SUBSTATE,
                                st->c33_ext_state_info.__c33_pse_ext_substate))
                        return -EMSGSIZE;
        }

        if (st->c33_avail_pw_limit > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT,
                        st->c33_avail_pw_limit))
                return -EMSGSIZE;

        if (st->c33_pw_limit_nb_ranges > 0 &&
            pse_put_pw_limit_ranges(skb, st))
                return -EMSGSIZE;

        return 0;
}

static void pse_cleanup_data(struct ethnl_reply_data *reply_base)
{
        const struct pse_reply_data *data = PSE_REPDATA(reply_base);

        kfree(data->status.c33_pw_limit_ranges);
}

/* PSE_SET */

const struct nla_policy ethnl_pse_set_policy[ETHTOOL_A_PSE_MAX + 1] = {
        [ETHTOOL_A_PSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy_phy),
        [ETHTOOL_A_PODL_PSE_ADMIN_CONTROL] =
                NLA_POLICY_RANGE(NLA_U32, ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED,
                                 ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED),
        [ETHTOOL_A_C33_PSE_ADMIN_CONTROL] =
                NLA_POLICY_RANGE(NLA_U32, ETHTOOL_C33_PSE_ADMIN_STATE_DISABLED,
                                 ETHTOOL_C33_PSE_ADMIN_STATE_ENABLED),
        [ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT] = { .type = NLA_U32 },
};

static int
ethnl_set_pse_validate(struct phy_device *phydev, struct genl_info *info)
{
        struct nlattr **tb = info->attrs;

        if (IS_ERR_OR_NULL(phydev)) {
                NL_SET_ERR_MSG(info->extack, "No PHY is attached");
                return -EOPNOTSUPP;
        }

        if (!phydev->psec) {
                NL_SET_ERR_MSG(info->extack, "No PSE is attached");
                return -EOPNOTSUPP;
        }

        if (tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL] &&
            !pse_has_podl(phydev->psec)) {
                NL_SET_ERR_MSG_ATTR(info->extack,
                                    tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL],
                                    "setting PoDL PSE admin control not supported");
                return -EOPNOTSUPP;
        }
        if (tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL] &&
            !pse_has_c33(phydev->psec)) {
                NL_SET_ERR_MSG_ATTR(info->extack,
                                    tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL],
                                    "setting C33 PSE admin control not supported");
                return -EOPNOTSUPP;
        }

        return 0;
}

static int
ethnl_set_pse(struct ethnl_req_info *req_info, struct genl_info *info)
{
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        int ret;

        phydev = ethnl_req_get_phydev(req_info, tb, ETHTOOL_A_PSE_HEADER,
                                      info->extack);
        ret = ethnl_set_pse_validate(phydev, info);
        if (ret)
                return ret;

        if (tb[ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT]) {
                unsigned int pw_limit;

                pw_limit = nla_get_u32(tb[ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT]);
                ret = pse_ethtool_set_pw_limit(phydev->psec, info->extack,
                                               pw_limit);
                if (ret)
                        return ret;
        }

        /* These values are already validated by the ethnl_pse_set_policy */
        if (tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL] ||
            tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL]) {
                struct pse_control_config config = {};

                if (tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL])
                        config.podl_admin_control = nla_get_u32(tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL]);
                if (tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL])
                        config.c33_admin_control = nla_get_u32(tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL]);

                /* pse_ethtool_set_config() will do nothing if the config
                 * is zero
                 */
                ret = pse_ethtool_set_config(phydev->psec, info->extack,
                                             &config);
                if (ret)
                        return ret;
        }

        /* Return errno or zero - PSE has no notification */
        return ret;
}

const struct ethnl_request_ops ethnl_pse_request_ops = {
        .request_cmd                = ETHTOOL_MSG_PSE_GET,
        .reply_cmd                = ETHTOOL_MSG_PSE_GET_REPLY,
        .hdr_attr                = ETHTOOL_A_PSE_HEADER,
        .req_info_size                = sizeof(struct pse_req_info),
        .reply_data_size        = sizeof(struct pse_reply_data),

        .prepare_data                = pse_prepare_data,
        .reply_size                = pse_reply_size,
        .fill_reply                = pse_fill_reply,
        .cleanup_data                = pse_cleanup_data,

        .set                        = ethnl_set_pse,
        /* PSE has no notification */
};










































































































    1 

    1 


    1 





    1 







    1 


    1 






    1 














































































































































































































































































































































































































































































































































    3 



    3 











    1 










    1 
    1 


    3 
    1 








    3 
























































    6 



    6 







    6 









    6 















    1 




















    6 




    7 
    3 





























    3 




    3 
    3 


    3 


    3 




























































    8 




























    8 



    8 
    8 












    8 




















    8 













    4 















    9 

    4 
    4 
    4 

    9 















    8 
    5 

    3 
    3 






    8 




    8 
    8 


































    8 














    8 
    3 




    8 




    8 
    3 

    5 

    8 
    8 




    8 


    8 















































































































    2 

































































    8 


    8 



    8 


    8 



    8 



    8 

    4 

















    8 



    8 


    8 



    8 





    8 




    8 






    8 

    8 
    8 







    1 


    1 



    1 
    1 






    1 

    1 







    1 

    1 
















    1 









    1 





    1 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
/*
   HIDP implementation for Linux Bluetooth stack (BlueZ).
   Copyright (C) 2003-2004 Marcel Holtmann <marcel@holtmann.org>
   Copyright (C) 2013 David Herrmann <dh.herrmann@gmail.com>

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License version 2 as
   published by the Free Software Foundation;

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
   OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS.
   IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) AND AUTHOR(S) BE LIABLE FOR ANY
   CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES
   WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
   ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
   OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

   ALL LIABILITY, INCLUDING LIABILITY FOR INFRINGEMENT OF ANY PATENTS,
   COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS, RELATING TO USE OF THIS
   SOFTWARE IS DISCLAIMED.
*/

#include <linux/kref.h>
#include <linux/module.h>
#include <linux/file.h>
#include <linux/kthread.h>
#include <linux/hidraw.h>

#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
#include <net/bluetooth/l2cap.h>

#include "hidp.h"

#define VERSION "1.2"

static DECLARE_RWSEM(hidp_session_sem);
static DECLARE_WAIT_QUEUE_HEAD(hidp_session_wq);
static LIST_HEAD(hidp_session_list);

static unsigned char hidp_keycode[256] = {
          0,   0,   0,   0,  30,  48,  46,  32,  18,  33,  34,  35,  23,  36,
         37,  38,  50,  49,  24,  25,  16,  19,  31,  20,  22,  47,  17,  45,
         21,  44,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  28,   1,
         14,  15,  57,  12,  13,  26,  27,  43,  43,  39,  40,  41,  51,  52,
         53,  58,  59,  60,  61,  62,  63,  64,  65,  66,  67,  68,  87,  88,
         99,  70, 119, 110, 102, 104, 111, 107, 109, 106, 105, 108, 103,  69,
         98,  55,  74,  78,  96,  79,  80,  81,  75,  76,  77,  71,  72,  73,
         82,  83,  86, 127, 116, 117, 183, 184, 185, 186, 187, 188, 189, 190,
        191, 192, 193, 194, 134, 138, 130, 132, 128, 129, 131, 137, 133, 135,
        136, 113, 115, 114,   0,   0,   0, 121,   0,  89,  93, 124,  92,  94,
         95,   0,   0,   0, 122, 123,  90,  91,  85,   0,   0,   0,   0,   0,
          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
         29,  42,  56, 125,  97,  54, 100, 126, 164, 166, 165, 163, 161, 115,
        114, 113, 150, 158, 159, 128, 136, 177, 178, 176, 142, 152, 173, 140
};

static unsigned char hidp_mkeyspat[] = { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 };

static int hidp_session_probe(struct l2cap_conn *conn,
                              struct l2cap_user *user);
static void hidp_session_remove(struct l2cap_conn *conn,
                                struct l2cap_user *user);
static int hidp_session_thread(void *arg);
static void hidp_session_terminate(struct hidp_session *s);

static void hidp_copy_session(struct hidp_session *session, struct hidp_conninfo *ci)
{
        u32 valid_flags = 0;
        memset(ci, 0, sizeof(*ci));
        bacpy(&ci->bdaddr, &session->bdaddr);

        ci->flags = session->flags & valid_flags;
        ci->state = BT_CONNECTED;

        if (session->input) {
                ci->vendor  = session->input->id.vendor;
                ci->product = session->input->id.product;
                ci->version = session->input->id.version;
                if (session->input->name)
                        strscpy(ci->name, session->input->name, 128);
                else
                        strscpy(ci->name, "HID Boot Device", 128);
        } else if (session->hid) {
                ci->vendor  = session->hid->vendor;
                ci->product = session->hid->product;
                ci->version = session->hid->version;
                strscpy(ci->name, session->hid->name, 128);
        }
}

/* assemble skb, queue message on @transmit and wake up the session thread */
static int hidp_send_message(struct hidp_session *session, struct socket *sock,
                             struct sk_buff_head *transmit, unsigned char hdr,
                             const unsigned char *data, int size)
{
        struct sk_buff *skb;
        struct sock *sk = sock->sk;
        int ret;

        BT_DBG("session %p data %p size %d", session, data, size);

        if (atomic_read(&session->terminate))
                return -EIO;

        skb = alloc_skb(size + 1, GFP_ATOMIC);
        if (!skb) {
                BT_ERR("Can't allocate memory for new frame");
                return -ENOMEM;
        }

        skb_put_u8(skb, hdr);
        if (data && size > 0) {
                skb_put_data(skb, data, size);
                ret = size;
        } else {
                ret = 0;
        }

        skb_queue_tail(transmit, skb);
        wake_up_interruptible(sk_sleep(sk));

        return ret;
}

static int hidp_send_ctrl_message(struct hidp_session *session,
                                  unsigned char hdr, const unsigned char *data,
                                  int size)
{
        return hidp_send_message(session, session->ctrl_sock,
                                 &session->ctrl_transmit, hdr, data, size);
}

static int hidp_send_intr_message(struct hidp_session *session,
                                  unsigned char hdr, const unsigned char *data,
                                  int size)
{
        return hidp_send_message(session, session->intr_sock,
                                 &session->intr_transmit, hdr, data, size);
}

static int hidp_input_event(struct input_dev *dev, unsigned int type,
                            unsigned int code, int value)
{
        struct hidp_session *session = input_get_drvdata(dev);
        unsigned char newleds;
        unsigned char hdr, data[2];

        BT_DBG("session %p type %d code %d value %d",
               session, type, code, value);

        if (type != EV_LED)
                return -1;

        newleds = (!!test_bit(LED_KANA,    dev->led) << 3) |
                  (!!test_bit(LED_COMPOSE, dev->led) << 3) |
                  (!!test_bit(LED_SCROLLL, dev->led) << 2) |
                  (!!test_bit(LED_CAPSL,   dev->led) << 1) |
                  (!!test_bit(LED_NUML,    dev->led) << 0);

        if (session->leds == newleds)
                return 0;

        session->leds = newleds;

        hdr = HIDP_TRANS_DATA | HIDP_DATA_RTYPE_OUPUT;
        data[0] = 0x01;
        data[1] = newleds;

        return hidp_send_intr_message(session, hdr, data, 2);
}

static void hidp_input_report(struct hidp_session *session, struct sk_buff *skb)
{
        struct input_dev *dev = session->input;
        unsigned char *keys = session->keys;
        unsigned char *udata = skb->data + 1;
        signed char *sdata = skb->data + 1;
        int i, size = skb->len - 1;

        switch (skb->data[0]) {
        case 0x01:        /* Keyboard report */
                for (i = 0; i < 8; i++)
                        input_report_key(dev, hidp_keycode[i + 224], (udata[0] >> i) & 1);

                /* If all the key codes have been set to 0x01, it means
                 * too many keys were pressed at the same time. */
                if (!memcmp(udata + 2, hidp_mkeyspat, 6))
                        break;

                for (i = 2; i < 8; i++) {
                        if (keys[i] > 3 && memscan(udata + 2, keys[i], 6) == udata + 8) {
                                if (hidp_keycode[keys[i]])
                                        input_report_key(dev, hidp_keycode[keys[i]], 0);
                                else
                                        BT_ERR("Unknown key (scancode %#x) released.", keys[i]);
                        }

                        if (udata[i] > 3 && memscan(keys + 2, udata[i], 6) == keys + 8) {
                                if (hidp_keycode[udata[i]])
                                        input_report_key(dev, hidp_keycode[udata[i]], 1);
                                else
                                        BT_ERR("Unknown key (scancode %#x) pressed.", udata[i]);
                        }
                }

                memcpy(keys, udata, 8);
                break;

        case 0x02:        /* Mouse report */
                input_report_key(dev, BTN_LEFT,   sdata[0] & 0x01);
                input_report_key(dev, BTN_RIGHT,  sdata[0] & 0x02);
                input_report_key(dev, BTN_MIDDLE, sdata[0] & 0x04);
                input_report_key(dev, BTN_SIDE,   sdata[0] & 0x08);
                input_report_key(dev, BTN_EXTRA,  sdata[0] & 0x10);

                input_report_rel(dev, REL_X, sdata[1]);
                input_report_rel(dev, REL_Y, sdata[2]);

                if (size > 3)
                        input_report_rel(dev, REL_WHEEL, sdata[3]);
                break;
        }

        input_sync(dev);
}

static int hidp_get_raw_report(struct hid_device *hid,
                unsigned char report_number,
                unsigned char *data, size_t count,
                unsigned char report_type)
{
        struct hidp_session *session = hid->driver_data;
        struct sk_buff *skb;
        size_t len;
        int numbered_reports = hid->report_enum[report_type].numbered;
        int ret;

        if (atomic_read(&session->terminate))
                return -EIO;

        switch (report_type) {
        case HID_FEATURE_REPORT:
                report_type = HIDP_TRANS_GET_REPORT | HIDP_DATA_RTYPE_FEATURE;
                break;
        case HID_INPUT_REPORT:
                report_type = HIDP_TRANS_GET_REPORT | HIDP_DATA_RTYPE_INPUT;
                break;
        case HID_OUTPUT_REPORT:
                report_type = HIDP_TRANS_GET_REPORT | HIDP_DATA_RTYPE_OUPUT;
                break;
        default:
                return -EINVAL;
        }

        if (mutex_lock_interruptible(&session->report_mutex))
                return -ERESTARTSYS;

        /* Set up our wait, and send the report request to the device. */
        session->waiting_report_type = report_type & HIDP_DATA_RTYPE_MASK;
        session->waiting_report_number = numbered_reports ? report_number : -1;
        set_bit(HIDP_WAITING_FOR_RETURN, &session->flags);
        data[0] = report_number;
        ret = hidp_send_ctrl_message(session, report_type, data, 1);
        if (ret < 0)
                goto err;

        /* Wait for the return of the report. The returned report
           gets put in session->report_return.  */
        while (test_bit(HIDP_WAITING_FOR_RETURN, &session->flags) &&
               !atomic_read(&session->terminate)) {
                int res;

                res = wait_event_interruptible_timeout(session->report_queue,
                        !test_bit(HIDP_WAITING_FOR_RETURN, &session->flags)
                                || atomic_read(&session->terminate),
                        5*HZ);
                if (res == 0) {
                        /* timeout */
                        ret = -EIO;
                        goto err;
                }
                if (res < 0) {
                        /* signal */
                        ret = -ERESTARTSYS;
                        goto err;
                }
        }

        skb = session->report_return;
        if (skb) {
                len = skb->len < count ? skb->len : count;
                memcpy(data, skb->data, len);

                kfree_skb(skb);
                session->report_return = NULL;
        } else {
                /* Device returned a HANDSHAKE, indicating  protocol error. */
                len = -EIO;
        }

        clear_bit(HIDP_WAITING_FOR_RETURN, &session->flags);
        mutex_unlock(&session->report_mutex);

        return len;

err:
        clear_bit(HIDP_WAITING_FOR_RETURN, &session->flags);
        mutex_unlock(&session->report_mutex);
        return ret;
}

static int hidp_set_raw_report(struct hid_device *hid, unsigned char reportnum,
                               unsigned char *data, size_t count,
                               unsigned char report_type)
{
        struct hidp_session *session = hid->driver_data;
        int ret;

        switch (report_type) {
        case HID_FEATURE_REPORT:
                report_type = HIDP_TRANS_SET_REPORT | HIDP_DATA_RTYPE_FEATURE;
                break;
        case HID_INPUT_REPORT:
                report_type = HIDP_TRANS_SET_REPORT | HIDP_DATA_RTYPE_INPUT;
                break;
        case HID_OUTPUT_REPORT:
                report_type = HIDP_TRANS_SET_REPORT | HIDP_DATA_RTYPE_OUPUT;
                break;
        default:
                return -EINVAL;
        }

        if (mutex_lock_interruptible(&session->report_mutex))
                return -ERESTARTSYS;

        /* Set up our wait, and send the report request to the device. */
        data[0] = reportnum;
        set_bit(HIDP_WAITING_FOR_SEND_ACK, &session->flags);
        ret = hidp_send_ctrl_message(session, report_type, data, count);
        if (ret < 0)
                goto err;

        /* Wait for the ACK from the device. */
        while (test_bit(HIDP_WAITING_FOR_SEND_ACK, &session->flags) &&
               !atomic_read(&session->terminate)) {
                int res;

                res = wait_event_interruptible_timeout(session->report_queue,
                        !test_bit(HIDP_WAITING_FOR_SEND_ACK, &session->flags)
                                || atomic_read(&session->terminate),
                        10*HZ);
                if (res == 0) {
                        /* timeout */
                        ret = -EIO;
                        goto err;
                }
                if (res < 0) {
                        /* signal */
                        ret = -ERESTARTSYS;
                        goto err;
                }
        }

        if (!session->output_report_success) {
                ret = -EIO;
                goto err;
        }

        ret = count;

err:
        clear_bit(HIDP_WAITING_FOR_SEND_ACK, &session->flags);
        mutex_unlock(&session->report_mutex);
        return ret;
}

static int hidp_output_report(struct hid_device *hid, __u8 *data, size_t count)
{
        struct hidp_session *session = hid->driver_data;

        return hidp_send_intr_message(session,
                                      HIDP_TRANS_DATA | HIDP_DATA_RTYPE_OUPUT,
                                      data, count);
}

static int hidp_raw_request(struct hid_device *hid, unsigned char reportnum,
                            __u8 *buf, size_t len, unsigned char rtype,
                            int reqtype)
{
        switch (reqtype) {
        case HID_REQ_GET_REPORT:
                return hidp_get_raw_report(hid, reportnum, buf, len, rtype);
        case HID_REQ_SET_REPORT:
                return hidp_set_raw_report(hid, reportnum, buf, len, rtype);
        default:
                return -EIO;
        }
}

static void hidp_idle_timeout(struct timer_list *t)
{
        struct hidp_session *session = from_timer(session, t, timer);

        /* The HIDP user-space API only contains calls to add and remove
         * devices. There is no way to forward events of any kind. Therefore,
         * we have to forcefully disconnect a device on idle-timeouts. This is
         * unfortunate and weird API design, but it is spec-compliant and
         * required for backwards-compatibility. Hence, on idle-timeout, we
         * signal driver-detach events, so poll() will be woken up with an
         * error-condition on both sockets.
         */

        session->intr_sock->sk->sk_err = EUNATCH;
        session->ctrl_sock->sk->sk_err = EUNATCH;
        wake_up_interruptible(sk_sleep(session->intr_sock->sk));
        wake_up_interruptible(sk_sleep(session->ctrl_sock->sk));

        hidp_session_terminate(session);
}

static void hidp_set_timer(struct hidp_session *session)
{
        if (session->idle_to > 0)
                mod_timer(&session->timer, jiffies + HZ * session->idle_to);
}

static void hidp_del_timer(struct hidp_session *session)
{
        if (session->idle_to > 0)
                del_timer_sync(&session->timer);
}

static void hidp_process_report(struct hidp_session *session, int type,
                                const u8 *data, unsigned int len, int intr)
{
        if (len > HID_MAX_BUFFER_SIZE)
                len = HID_MAX_BUFFER_SIZE;

        memcpy(session->input_buf, data, len);
        hid_input_report(session->hid, type, session->input_buf, len, intr);
}

static void hidp_process_handshake(struct hidp_session *session,
                                        unsigned char param)
{
        BT_DBG("session %p param 0x%02x", session, param);
        session->output_report_success = 0; /* default condition */

        switch (param) {
        case HIDP_HSHK_SUCCESSFUL:
                /* FIXME: Call into SET_ GET_ handlers here */
                session->output_report_success = 1;
                break;

        case HIDP_HSHK_NOT_READY:
        case HIDP_HSHK_ERR_INVALID_REPORT_ID:
        case HIDP_HSHK_ERR_UNSUPPORTED_REQUEST:
        case HIDP_HSHK_ERR_INVALID_PARAMETER:
                if (test_and_clear_bit(HIDP_WAITING_FOR_RETURN, &session->flags))
                        wake_up_interruptible(&session->report_queue);

                /* FIXME: Call into SET_ GET_ handlers here */
                break;

        case HIDP_HSHK_ERR_UNKNOWN:
                break;

        case HIDP_HSHK_ERR_FATAL:
                /* Device requests a reboot, as this is the only way this error
                 * can be recovered. */
                hidp_send_ctrl_message(session,
                        HIDP_TRANS_HID_CONTROL | HIDP_CTRL_SOFT_RESET, NULL, 0);
                break;

        default:
                hidp_send_ctrl_message(session,
                        HIDP_TRANS_HANDSHAKE | HIDP_HSHK_ERR_INVALID_PARAMETER, NULL, 0);
                break;
        }

        /* Wake up the waiting thread. */
        if (test_and_clear_bit(HIDP_WAITING_FOR_SEND_ACK, &session->flags))
                wake_up_interruptible(&session->report_queue);
}

static void hidp_process_hid_control(struct hidp_session *session,
                                        unsigned char param)
{
        BT_DBG("session %p param 0x%02x", session, param);

        if (param == HIDP_CTRL_VIRTUAL_CABLE_UNPLUG) {
                /* Flush the transmit queues */
                skb_queue_purge(&session->ctrl_transmit);
                skb_queue_purge(&session->intr_transmit);

                hidp_session_terminate(session);
        }
}

/* Returns true if the passed-in skb should be freed by the caller. */
static int hidp_process_data(struct hidp_session *session, struct sk_buff *skb,
                                unsigned char param)
{
        int done_with_skb = 1;
        BT_DBG("session %p skb %p len %u param 0x%02x", session, skb, skb->len, param);

        switch (param) {
        case HIDP_DATA_RTYPE_INPUT:
                hidp_set_timer(session);

                if (session->input)
                        hidp_input_report(session, skb);

                if (session->hid)
                        hidp_process_report(session, HID_INPUT_REPORT,
                                            skb->data, skb->len, 0);
                break;

        case HIDP_DATA_RTYPE_OTHER:
        case HIDP_DATA_RTYPE_OUPUT:
        case HIDP_DATA_RTYPE_FEATURE:
                break;

        default:
                hidp_send_ctrl_message(session,
                        HIDP_TRANS_HANDSHAKE | HIDP_HSHK_ERR_INVALID_PARAMETER, NULL, 0);
        }

        if (test_bit(HIDP_WAITING_FOR_RETURN, &session->flags) &&
                                param == session->waiting_report_type) {
                if (session->waiting_report_number < 0 ||
                    session->waiting_report_number == skb->data[0]) {
                        /* hidp_get_raw_report() is waiting on this report. */
                        session->report_return = skb;
                        done_with_skb = 0;
                        clear_bit(HIDP_WAITING_FOR_RETURN, &session->flags);
                        wake_up_interruptible(&session->report_queue);
                }
        }

        return done_with_skb;
}

static void hidp_recv_ctrl_frame(struct hidp_session *session,
                                        struct sk_buff *skb)
{
        unsigned char hdr, type, param;
        int free_skb = 1;

        BT_DBG("session %p skb %p len %u", session, skb, skb->len);

        hdr = skb->data[0];
        skb_pull(skb, 1);

        type = hdr & HIDP_HEADER_TRANS_MASK;
        param = hdr & HIDP_HEADER_PARAM_MASK;

        switch (type) {
        case HIDP_TRANS_HANDSHAKE:
                hidp_process_handshake(session, param);
                break;

        case HIDP_TRANS_HID_CONTROL:
                hidp_process_hid_control(session, param);
                break;

        case HIDP_TRANS_DATA:
                free_skb = hidp_process_data(session, skb, param);
                break;

        default:
                hidp_send_ctrl_message(session,
                        HIDP_TRANS_HANDSHAKE | HIDP_HSHK_ERR_UNSUPPORTED_REQUEST, NULL, 0);
                break;
        }

        if (free_skb)
                kfree_skb(skb);
}

static void hidp_recv_intr_frame(struct hidp_session *session,
                                struct sk_buff *skb)
{
        unsigned char hdr;

        BT_DBG("session %p skb %p len %u", session, skb, skb->len);

        hdr = skb->data[0];
        skb_pull(skb, 1);

        if (hdr == (HIDP_TRANS_DATA | HIDP_DATA_RTYPE_INPUT)) {
                hidp_set_timer(session);

                if (session->input)
                        hidp_input_report(session, skb);

                if (session->hid) {
                        hidp_process_report(session, HID_INPUT_REPORT,
                                            skb->data, skb->len, 1);
                        BT_DBG("report len %d", skb->len);
                }
        } else {
                BT_DBG("Unsupported protocol header 0x%02x", hdr);
        }

        kfree_skb(skb);
}

static int hidp_send_frame(struct socket *sock, unsigned char *data, int len)
{
        struct kvec iv = { data, len };
        struct msghdr msg;

        BT_DBG("sock %p data %p len %d", sock, data, len);

        if (!len)
                return 0;

        memset(&msg, 0, sizeof(msg));

        return kernel_sendmsg(sock, &msg, &iv, 1, len);
}

/* dequeue message from @transmit and send via @sock */
static void hidp_process_transmit(struct hidp_session *session,
                                  struct sk_buff_head *transmit,
                                  struct socket *sock)
{
        struct sk_buff *skb;
        int ret;

        BT_DBG("session %p", session);

        while ((skb = skb_dequeue(transmit))) {
                ret = hidp_send_frame(sock, skb->data, skb->len);
                if (ret == -EAGAIN) {
                        skb_queue_head(transmit, skb);
                        break;
                } else if (ret < 0) {
                        hidp_session_terminate(session);
                        kfree_skb(skb);
                        break;
                }

                hidp_set_timer(session);
                kfree_skb(skb);
        }
}

static int hidp_setup_input(struct hidp_session *session,
                                const struct hidp_connadd_req *req)
{
        struct input_dev *input;
        int i;

        input = input_allocate_device();
        if (!input)
                return -ENOMEM;

        session->input = input;

        input_set_drvdata(input, session);

        input->name = "Bluetooth HID Boot Protocol Device";

        input->id.bustype = BUS_BLUETOOTH;
        input->id.vendor  = req->vendor;
        input->id.product = req->product;
        input->id.version = req->version;

        if (req->subclass & 0x40) {
                set_bit(EV_KEY, input->evbit);
                set_bit(EV_LED, input->evbit);
                set_bit(EV_REP, input->evbit);

                set_bit(LED_NUML,    input->ledbit);
                set_bit(LED_CAPSL,   input->ledbit);
                set_bit(LED_SCROLLL, input->ledbit);
                set_bit(LED_COMPOSE, input->ledbit);
                set_bit(LED_KANA,    input->ledbit);

                for (i = 0; i < sizeof(hidp_keycode); i++)
                        set_bit(hidp_keycode[i], input->keybit);
                clear_bit(0, input->keybit);
        }

        if (req->subclass & 0x80) {
                input->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_REL);
                input->keybit[BIT_WORD(BTN_MOUSE)] = BIT_MASK(BTN_LEFT) |
                        BIT_MASK(BTN_RIGHT) | BIT_MASK(BTN_MIDDLE);
                input->relbit[0] = BIT_MASK(REL_X) | BIT_MASK(REL_Y);
                input->keybit[BIT_WORD(BTN_MOUSE)] |= BIT_MASK(BTN_SIDE) |
                        BIT_MASK(BTN_EXTRA);
                input->relbit[0] |= BIT_MASK(REL_WHEEL);
        }

        input->dev.parent = &session->conn->hcon->dev;

        input->event = hidp_input_event;

        return 0;
}

static int hidp_open(struct hid_device *hid)
{
        return 0;
}

static void hidp_close(struct hid_device *hid)
{
}

static int hidp_parse(struct hid_device *hid)
{
        struct hidp_session *session = hid->driver_data;

        return hid_parse_report(session->hid, session->rd_data,
                        session->rd_size);
}

static int hidp_start(struct hid_device *hid)
{
        return 0;
}

static void hidp_stop(struct hid_device *hid)
{
        struct hidp_session *session = hid->driver_data;

        skb_queue_purge(&session->ctrl_transmit);
        skb_queue_purge(&session->intr_transmit);

        hid->claimed = 0;
}

static const struct hid_ll_driver hidp_hid_driver = {
        .parse = hidp_parse,
        .start = hidp_start,
        .stop = hidp_stop,
        .open  = hidp_open,
        .close = hidp_close,
        .raw_request = hidp_raw_request,
        .output_report = hidp_output_report,
};

/* This function sets up the hid device. It does not add it
   to the HID system. That is done in hidp_add_connection(). */
static int hidp_setup_hid(struct hidp_session *session,
                                const struct hidp_connadd_req *req)
{
        struct hid_device *hid;
        int err;

        session->rd_data = memdup_user(req->rd_data, req->rd_size);
        if (IS_ERR(session->rd_data))
                return PTR_ERR(session->rd_data);

        session->rd_size = req->rd_size;

        hid = hid_allocate_device();
        if (IS_ERR(hid)) {
                err = PTR_ERR(hid);
                goto fault;
        }

        session->hid = hid;

        hid->driver_data = session;

        hid->bus     = BUS_BLUETOOTH;
        hid->vendor  = req->vendor;
        hid->product = req->product;
        hid->version = req->version;
        hid->country = req->country;

        strscpy(hid->name, req->name, sizeof(hid->name));

        snprintf(hid->phys, sizeof(hid->phys), "%pMR",
                 &l2cap_pi(session->ctrl_sock->sk)->chan->src);

        /* NOTE: Some device modules depend on the dst address being stored in
         * uniq. Please be aware of this before making changes to this behavior.
         */
        snprintf(hid->uniq, sizeof(hid->uniq), "%pMR",
                 &l2cap_pi(session->ctrl_sock->sk)->chan->dst);

        hid->dev.parent = &session->conn->hcon->dev;
        hid->ll_driver = &hidp_hid_driver;

        /* True if device is blocked in drivers/hid/hid-quirks.c */
        if (hid_ignore(hid)) {
                hid_destroy_device(session->hid);
                session->hid = NULL;
                return -ENODEV;
        }

        return 0;

fault:
        kfree(session->rd_data);
        session->rd_data = NULL;

        return err;
}

/* initialize session devices */
static int hidp_session_dev_init(struct hidp_session *session,
                                 const struct hidp_connadd_req *req)
{
        int ret;

        if (req->rd_size > 0) {
                ret = hidp_setup_hid(session, req);
                if (ret && ret != -ENODEV)
                        return ret;
        }

        if (!session->hid) {
                ret = hidp_setup_input(session, req);
                if (ret < 0)
                        return ret;
        }

        return 0;
}

/* destroy session devices */
static void hidp_session_dev_destroy(struct hidp_session *session)
{
        if (session->hid)
                put_device(&session->hid->dev);
        else if (session->input)
                input_put_device(session->input);

        kfree(session->rd_data);
        session->rd_data = NULL;
}

/* add HID/input devices to their underlying bus systems */
static int hidp_session_dev_add(struct hidp_session *session)
{
        int ret;

        /* Both HID and input systems drop a ref-count when unregistering the
         * device but they don't take a ref-count when registering them. Work
         * around this by explicitly taking a refcount during registration
         * which is dropped automatically by unregistering the devices. */

        if (session->hid) {
                ret = hid_add_device(session->hid);
                if (ret)
                        return ret;
                get_device(&session->hid->dev);
        } else if (session->input) {
                ret = input_register_device(session->input);
                if (ret)
                        return ret;
                input_get_device(session->input);
        }

        return 0;
}

/* remove HID/input devices from their bus systems */
static void hidp_session_dev_del(struct hidp_session *session)
{
        if (session->hid)
                hid_destroy_device(session->hid);
        else if (session->input)
                input_unregister_device(session->input);
}

/*
 * Asynchronous device registration
 * HID device drivers might want to perform I/O during initialization to
 * detect device types. Therefore, call device registration in a separate
 * worker so the HIDP thread can schedule I/O operations.
 * Note that this must be called after the worker thread was initialized
 * successfully. This will then add the devices and increase session state
 * on success, otherwise it will terminate the session thread.
 */
static void hidp_session_dev_work(struct work_struct *work)
{
        struct hidp_session *session = container_of(work,
                                                    struct hidp_session,
                                                    dev_init);
        int ret;

        ret = hidp_session_dev_add(session);
        if (!ret)
                atomic_inc(&session->state);
        else
                hidp_session_terminate(session);
}

/*
 * Create new session object
 * Allocate session object, initialize static fields, copy input data into the
 * object and take a reference to all sub-objects.
 * This returns 0 on success and puts a pointer to the new session object in
 * \out. Otherwise, an error code is returned.
 * The new session object has an initial ref-count of 1.
 */
static int hidp_session_new(struct hidp_session **out, const bdaddr_t *bdaddr,
                            struct socket *ctrl_sock,
                            struct socket *intr_sock,
                            const struct hidp_connadd_req *req,
                            struct l2cap_conn *conn)
{
        struct hidp_session *session;
        int ret;
        struct bt_sock *ctrl, *intr;

        ctrl = bt_sk(ctrl_sock->sk);
        intr = bt_sk(intr_sock->sk);

        session = kzalloc(sizeof(*session), GFP_KERNEL);
        if (!session)
                return -ENOMEM;

        /* object and runtime management */
        kref_init(&session->ref);
        atomic_set(&session->state, HIDP_SESSION_IDLING);
        init_waitqueue_head(&session->state_queue);
        session->flags = req->flags & BIT(HIDP_BLUETOOTH_VENDOR_ID);

        /* connection management */
        bacpy(&session->bdaddr, bdaddr);
        session->conn = l2cap_conn_get(conn);
        session->user.probe = hidp_session_probe;
        session->user.remove = hidp_session_remove;
        INIT_LIST_HEAD(&session->user.list);
        session->ctrl_sock = ctrl_sock;
        session->intr_sock = intr_sock;
        skb_queue_head_init(&session->ctrl_transmit);
        skb_queue_head_init(&session->intr_transmit);
        session->ctrl_mtu = min_t(uint, l2cap_pi(ctrl)->chan->omtu,
                                        l2cap_pi(ctrl)->chan->imtu);
        session->intr_mtu = min_t(uint, l2cap_pi(intr)->chan->omtu,
                                        l2cap_pi(intr)->chan->imtu);
        session->idle_to = req->idle_to;

        /* device management */
        INIT_WORK(&session->dev_init, hidp_session_dev_work);
        timer_setup(&session->timer, hidp_idle_timeout, 0);

        /* session data */
        mutex_init(&session->report_mutex);
        init_waitqueue_head(&session->report_queue);

        ret = hidp_session_dev_init(session, req);
        if (ret)
                goto err_free;

        get_file(session->intr_sock->file);
        get_file(session->ctrl_sock->file);
        *out = session;
        return 0;

err_free:
        l2cap_conn_put(session->conn);
        kfree(session);
        return ret;
}

/* increase ref-count of the given session by one */
static void hidp_session_get(struct hidp_session *session)
{
        kref_get(&session->ref);
}

/* release callback */
static void session_free(struct kref *ref)
{
        struct hidp_session *session = container_of(ref, struct hidp_session,
                                                    ref);

        hidp_session_dev_destroy(session);
        skb_queue_purge(&session->ctrl_transmit);
        skb_queue_purge(&session->intr_transmit);
        fput(session->intr_sock->file);
        fput(session->ctrl_sock->file);
        l2cap_conn_put(session->conn);
        kfree(session);
}

/* decrease ref-count of the given session by one */
static void hidp_session_put(struct hidp_session *session)
{
        kref_put(&session->ref, session_free);
}

/*
 * Search the list of active sessions for a session with target address
 * \bdaddr. You must hold at least a read-lock on \hidp_session_sem. As long as
 * you do not release this lock, the session objects cannot vanish and you can
 * safely take a reference to the session yourself.
 */
static struct hidp_session *__hidp_session_find(const bdaddr_t *bdaddr)
{
        struct hidp_session *session;

        list_for_each_entry(session, &hidp_session_list, list) {
                if (!bacmp(bdaddr, &session->bdaddr))
                        return session;
        }

        return NULL;
}

/*
 * Same as __hidp_session_find() but no locks must be held. This also takes a
 * reference of the returned session (if non-NULL) so you must drop this
 * reference if you no longer use the object.
 */
static struct hidp_session *hidp_session_find(const bdaddr_t *bdaddr)
{
        struct hidp_session *session;

        down_read(&hidp_session_sem);

        session = __hidp_session_find(bdaddr);
        if (session)
                hidp_session_get(session);

        up_read(&hidp_session_sem);

        return session;
}

/*
 * Start session synchronously
 * This starts a session thread and waits until initialization
 * is done or returns an error if it couldn't be started.
 * If this returns 0 the session thread is up and running. You must call
 * hipd_session_stop_sync() before deleting any runtime resources.
 */
static int hidp_session_start_sync(struct hidp_session *session)
{
        unsigned int vendor, product;

        if (session->hid) {
                vendor  = session->hid->vendor;
                product = session->hid->product;
        } else if (session->input) {
                vendor  = session->input->id.vendor;
                product = session->input->id.product;
        } else {
                vendor = 0x0000;
                product = 0x0000;
        }

        session->task = kthread_run(hidp_session_thread, session,
                                    "khidpd_%04x%04x", vendor, product);
        if (IS_ERR(session->task))
                return PTR_ERR(session->task);

        while (atomic_read(&session->state) <= HIDP_SESSION_IDLING)
                wait_event(session->state_queue,
                           atomic_read(&session->state) > HIDP_SESSION_IDLING);

        return 0;
}

/*
 * Terminate session thread
 * Wake up session thread and notify it to stop. This is asynchronous and
 * returns immediately. Call this whenever a runtime error occurs and you want
 * the session to stop.
 * Note: wake_up_interruptible() performs any necessary memory-barriers for us.
 */
static void hidp_session_terminate(struct hidp_session *session)
{
        atomic_inc(&session->terminate);
        /*
         * See the comment preceding the call to wait_woken()
         * in hidp_session_run().
         */
        wake_up_interruptible(&hidp_session_wq);
}

/*
 * Probe HIDP session
 * This is called from the l2cap_conn core when our l2cap_user object is bound
 * to the hci-connection. We get the session via the \user object and can now
 * start the session thread, link it into the global session list and
 * schedule HID/input device registration.
 * The global session-list owns its own reference to the session object so you
 * can drop your own reference after registering the l2cap_user object.
 */
static int hidp_session_probe(struct l2cap_conn *conn,
                              struct l2cap_user *user)
{
        struct hidp_session *session = container_of(user,
                                                    struct hidp_session,
                                                    user);
        struct hidp_session *s;
        int ret;

        down_write(&hidp_session_sem);

        /* check that no other session for this device exists */
        s = __hidp_session_find(&session->bdaddr);
        if (s) {
                ret = -EEXIST;
                goto out_unlock;
        }

        if (session->input) {
                ret = hidp_session_dev_add(session);
                if (ret)
                        goto out_unlock;
        }

        ret = hidp_session_start_sync(session);
        if (ret)
                goto out_del;

        /* HID device registration is async to allow I/O during probe */
        if (session->input)
                atomic_inc(&session->state);
        else
                schedule_work(&session->dev_init);

        hidp_session_get(session);
        list_add(&session->list, &hidp_session_list);
        ret = 0;
        goto out_unlock;

out_del:
        if (session->input)
                hidp_session_dev_del(session);
out_unlock:
        up_write(&hidp_session_sem);
        return ret;
}

/*
 * Remove HIDP session
 * Called from the l2cap_conn core when either we explicitly unregistered
 * the l2cap_user object or if the underlying connection is shut down.
 * We signal the hidp-session thread to shut down, unregister the HID/input
 * devices and unlink the session from the global list.
 * This drops the reference to the session that is owned by the global
 * session-list.
 * Note: We _must_ not synchronosly wait for the session-thread to shut down.
 * This is, because the session-thread might be waiting for an HCI lock that is
 * held while we are called. Therefore, we only unregister the devices and
 * notify the session-thread to terminate. The thread itself owns a reference
 * to the session object so it can safely shut down.
 */
static void hidp_session_remove(struct l2cap_conn *conn,
                                struct l2cap_user *user)
{
        struct hidp_session *session = container_of(user,
                                                    struct hidp_session,
                                                    user);

        down_write(&hidp_session_sem);

        hidp_session_terminate(session);

        cancel_work_sync(&session->dev_init);
        if (session->input ||
            atomic_read(&session->state) > HIDP_SESSION_PREPARING)
                hidp_session_dev_del(session);

        list_del(&session->list);

        up_write(&hidp_session_sem);

        hidp_session_put(session);
}

/*
 * Session Worker
 * This performs the actual main-loop of the HIDP worker. We first check
 * whether the underlying connection is still alive, then parse all pending
 * messages and finally send all outstanding messages.
 */
static void hidp_session_run(struct hidp_session *session)
{
        struct sock *ctrl_sk = session->ctrl_sock->sk;
        struct sock *intr_sk = session->intr_sock->sk;
        struct sk_buff *skb;
        DEFINE_WAIT_FUNC(wait, woken_wake_function);

        add_wait_queue(&hidp_session_wq, &wait);
        for (;;) {
                /*
                 * This thread can be woken up two ways:
                 *  - You call hidp_session_terminate() which sets the
                 *    session->terminate flag and wakes this thread up.
                 *  - Via modifying the socket state of ctrl/intr_sock. This
                 *    thread is woken up by ->sk_state_changed().
                 */

                if (atomic_read(&session->terminate))
                        break;

                if (ctrl_sk->sk_state != BT_CONNECTED ||
                    intr_sk->sk_state != BT_CONNECTED)
                        break;

                /* parse incoming intr-skbs */
                while ((skb = skb_dequeue(&intr_sk->sk_receive_queue))) {
                        skb_orphan(skb);
                        if (!skb_linearize(skb))
                                hidp_recv_intr_frame(session, skb);
                        else
                                kfree_skb(skb);
                }

                /* send pending intr-skbs */
                hidp_process_transmit(session, &session->intr_transmit,
                                      session->intr_sock);

                /* parse incoming ctrl-skbs */
                while ((skb = skb_dequeue(&ctrl_sk->sk_receive_queue))) {
                        skb_orphan(skb);
                        if (!skb_linearize(skb))
                                hidp_recv_ctrl_frame(session, skb);
                        else
                                kfree_skb(skb);
                }

                /* send pending ctrl-skbs */
                hidp_process_transmit(session, &session->ctrl_transmit,
                                      session->ctrl_sock);

                /*
                 * wait_woken() performs the necessary memory barriers
                 * for us; see the header comment for this primitive.
                 */
                wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
        }
        remove_wait_queue(&hidp_session_wq, &wait);

        atomic_inc(&session->terminate);
}

static int hidp_session_wake_function(wait_queue_entry_t *wait,
                                      unsigned int mode,
                                      int sync, void *key)
{
        wake_up_interruptible(&hidp_session_wq);
        return false;
}

/*
 * HIDP session thread
 * This thread runs the I/O for a single HIDP session. Startup is synchronous
 * which allows us to take references to ourself here instead of doing that in
 * the caller.
 * When we are ready to run we notify the caller and call hidp_session_run().
 */
static int hidp_session_thread(void *arg)
{
        struct hidp_session *session = arg;
        DEFINE_WAIT_FUNC(ctrl_wait, hidp_session_wake_function);
        DEFINE_WAIT_FUNC(intr_wait, hidp_session_wake_function);

        BT_DBG("session %p", session);

        /* initialize runtime environment */
        hidp_session_get(session);
        __module_get(THIS_MODULE);
        set_user_nice(current, -15);
        hidp_set_timer(session);

        add_wait_queue(sk_sleep(session->ctrl_sock->sk), &ctrl_wait);
        add_wait_queue(sk_sleep(session->intr_sock->sk), &intr_wait);
        /* This memory barrier is paired with wq_has_sleeper(). See
         * sock_poll_wait() for more information why this is needed. */
        smp_mb__before_atomic();

        /* notify synchronous startup that we're ready */
        atomic_inc(&session->state);
        wake_up(&session->state_queue);

        /* run session */
        hidp_session_run(session);

        /* cleanup runtime environment */
        remove_wait_queue(sk_sleep(session->intr_sock->sk), &intr_wait);
        remove_wait_queue(sk_sleep(session->ctrl_sock->sk), &ctrl_wait);
        wake_up_interruptible(&session->report_queue);
        hidp_del_timer(session);

        /*
         * If we stopped ourself due to any internal signal, we should try to
         * unregister our own session here to avoid having it linger until the
         * parent l2cap_conn dies or user-space cleans it up.
         * This does not deadlock as we don't do any synchronous shutdown.
         * Instead, this call has the same semantics as if user-space tried to
         * delete the session.
         */
        l2cap_unregister_user(session->conn, &session->user);
        hidp_session_put(session);

        module_put_and_kthread_exit(0);
        return 0;
}

static int hidp_verify_sockets(struct socket *ctrl_sock,
                               struct socket *intr_sock)
{
        struct l2cap_chan *ctrl_chan, *intr_chan;
        struct bt_sock *ctrl, *intr;
        struct hidp_session *session;

        if (!l2cap_is_socket(ctrl_sock) || !l2cap_is_socket(intr_sock))
                return -EINVAL;

        ctrl_chan = l2cap_pi(ctrl_sock->sk)->chan;
        intr_chan = l2cap_pi(intr_sock->sk)->chan;

        if (bacmp(&ctrl_chan->src, &intr_chan->src) ||
            bacmp(&ctrl_chan->dst, &intr_chan->dst))
                return -ENOTUNIQ;

        ctrl = bt_sk(ctrl_sock->sk);
        intr = bt_sk(intr_sock->sk);

        if (ctrl->sk.sk_state != BT_CONNECTED ||
            intr->sk.sk_state != BT_CONNECTED)
                return -EBADFD;

        /* early session check, we check again during session registration */
        session = hidp_session_find(&ctrl_chan->dst);
        if (session) {
                hidp_session_put(session);
                return -EEXIST;
        }

        return 0;
}

int hidp_connection_add(const struct hidp_connadd_req *req,
                        struct socket *ctrl_sock,
                        struct socket *intr_sock)
{
        u32 valid_flags = BIT(HIDP_VIRTUAL_CABLE_UNPLUG) |
                          BIT(HIDP_BOOT_PROTOCOL_MODE);
        struct hidp_session *session;
        struct l2cap_conn *conn;
        struct l2cap_chan *chan;
        int ret;

        ret = hidp_verify_sockets(ctrl_sock, intr_sock);
        if (ret)
                return ret;

        if (req->flags & ~valid_flags)
                return -EINVAL;

        chan = l2cap_pi(ctrl_sock->sk)->chan;
        conn = NULL;
        l2cap_chan_lock(chan);
        if (chan->conn)
                conn = l2cap_conn_get(chan->conn);
        l2cap_chan_unlock(chan);

        if (!conn)
                return -EBADFD;

        ret = hidp_session_new(&session, &chan->dst, ctrl_sock,
                               intr_sock, req, conn);
        if (ret)
                goto out_conn;

        ret = l2cap_register_user(conn, &session->user);
        if (ret)
                goto out_session;

        ret = 0;

out_session:
        hidp_session_put(session);
out_conn:
        l2cap_conn_put(conn);
        return ret;
}

int hidp_connection_del(struct hidp_conndel_req *req)
{
        u32 valid_flags = BIT(HIDP_VIRTUAL_CABLE_UNPLUG);
        struct hidp_session *session;

        if (req->flags & ~valid_flags)
                return -EINVAL;

        session = hidp_session_find(&req->bdaddr);
        if (!session)
                return -ENOENT;

        if (req->flags & BIT(HIDP_VIRTUAL_CABLE_UNPLUG))
                hidp_send_ctrl_message(session,
                                       HIDP_TRANS_HID_CONTROL |
                                         HIDP_CTRL_VIRTUAL_CABLE_UNPLUG,
                                       NULL, 0);
        else
                l2cap_unregister_user(session->conn, &session->user);

        hidp_session_put(session);

        return 0;
}

int hidp_get_connlist(struct hidp_connlist_req *req)
{
        struct hidp_session *session;
        int err = 0, n = 0;

        BT_DBG("");

        down_read(&hidp_session_sem);

        list_for_each_entry(session, &hidp_session_list, list) {
                struct hidp_conninfo ci;

                hidp_copy_session(session, &ci);

                if (copy_to_user(req->ci, &ci, sizeof(ci))) {
                        err = -EFAULT;
                        break;
                }

                if (++n >= req->cnum)
                        break;

                req->ci++;
        }
        req->cnum = n;

        up_read(&hidp_session_sem);
        return err;
}

int hidp_get_conninfo(struct hidp_conninfo *ci)
{
        struct hidp_session *session;

        session = hidp_session_find(&ci->bdaddr);
        if (session) {
                hidp_copy_session(session, ci);
                hidp_session_put(session);
        }

        return session ? 0 : -ENOENT;
}

static int __init hidp_init(void)
{
        BT_INFO("HIDP (Human Interface Emulation) ver %s", VERSION);

        return hidp_init_sockets();
}

static void __exit hidp_exit(void)
{
        hidp_cleanup_sockets();
}

module_init(hidp_init);
module_exit(hidp_exit);

MODULE_AUTHOR("Marcel Holtmann <marcel@holtmann.org>");
MODULE_AUTHOR("David Herrmann <dh.herrmann@gmail.com>");
MODULE_DESCRIPTION("Bluetooth HIDP ver " VERSION);
MODULE_VERSION(VERSION);
MODULE_LICENSE("GPL");
MODULE_ALIAS("bt-proto-6");


















































































































































    3 






















































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _FAT_H
#define _FAT_H

#include <linux/buffer_head.h>
#include <linux/nls.h>
#include <linux/hash.h>
#include <linux/ratelimit.h>
#include <linux/msdos_fs.h>
#include <linux/fs_context.h>
#include <linux/fs_parser.h>

/*
 * vfat shortname flags
 */
#define VFAT_SFN_DISPLAY_LOWER        0x0001 /* convert to lowercase for display */
#define VFAT_SFN_DISPLAY_WIN95        0x0002 /* emulate win95 rule for display */
#define VFAT_SFN_DISPLAY_WINNT        0x0004 /* emulate winnt rule for display */
#define VFAT_SFN_CREATE_WIN95        0x0100 /* emulate win95 rule for create */
#define VFAT_SFN_CREATE_WINNT        0x0200 /* emulate winnt rule for create */

#define FAT_ERRORS_CONT                1      /* ignore error and continue */
#define FAT_ERRORS_PANIC        2      /* panic on error */
#define FAT_ERRORS_RO                3      /* remount r/o on error */

#define FAT_NFS_STALE_RW        1      /* NFS RW support, can cause ESTALE */
#define FAT_NFS_NOSTALE_RO        2      /* NFS RO support, no ESTALE issue */

struct fat_mount_options {
        kuid_t fs_uid;
        kgid_t fs_gid;
        unsigned short fs_fmask;
        unsigned short fs_dmask;
        unsigned short codepage;   /* Codepage for shortname conversions */
        int time_offset;           /* Offset of timestamps from UTC (in minutes) */
        char *iocharset;           /* Charset used for filename input/display */
        unsigned short shortname;  /* flags for shortname display/create rule */
        unsigned char name_check;  /* r = relaxed, n = normal, s = strict */
        unsigned char errors;           /* On error: continue, panic, remount-ro */
        unsigned char nfs;          /* NFS support: nostale_ro, stale_rw */
        unsigned short allow_utime;/* permission for setting the [am]time */
        unsigned quiet:1,          /* set = fake successful chmods and chowns */
                 showexec:1,       /* set = only set x bit for com/exe/bat */
                 sys_immutable:1,  /* set = system files are immutable */
                 dotsOK:1,         /* set = hidden and system files are named '.filename' */
                 isvfat:1,         /* 0=no vfat long filename support, 1=vfat support */
                 utf8:1,           /* Use of UTF-8 character set (Default) */
                 unicode_xlate:1,  /* create escape sequences for unhandled Unicode */
                 numtail:1,        /* Does first alias have a numeric '~1' type tail? */
                 flush:1,           /* write things quickly */
                 nocase:1,           /* Does this need case conversion? 0=need case conversion*/
                 usefree:1,           /* Use free_clusters for FAT32 */
                 tz_set:1,           /* Filesystem timestamps' offset set */
                 rodir:1,           /* allow ATTR_RO for directory */
                 discard:1,           /* Issue discard requests on deletions */
                 dos1xfloppy:1,           /* Assume default BPB for DOS 1.x floppies */
                 debug:1;           /* Not currently used */
};

#define FAT_HASH_BITS        8
#define FAT_HASH_SIZE        (1UL << FAT_HASH_BITS)

/*
 * MS-DOS file system in-core superblock data
 */
struct msdos_sb_info {
        unsigned short sec_per_clus;  /* sectors/cluster */
        unsigned short cluster_bits;  /* log2(cluster_size) */
        unsigned int cluster_size;    /* cluster size */
        unsigned char fats, fat_bits; /* number of FATs, FAT bits (12,16 or 32) */
        unsigned short fat_start;
        unsigned long fat_length;     /* FAT start & length (sec.) */
        unsigned long dir_start;
        unsigned short dir_entries;   /* root dir start & entries */
        unsigned long data_start;     /* first data sector */
        unsigned long max_cluster;    /* maximum cluster number */
        unsigned long root_cluster;   /* first cluster of the root directory */
        unsigned long fsinfo_sector;  /* sector number of FAT32 fsinfo */
        struct mutex fat_lock;
        struct mutex nfs_build_inode_lock;
        struct mutex s_lock;
        unsigned int prev_free;      /* previously allocated cluster number */
        unsigned int free_clusters;  /* -1 if undefined */
        unsigned int free_clus_valid; /* is free_clusters valid? */
        struct fat_mount_options options;
        struct nls_table *nls_disk;   /* Codepage used on disk */
        struct nls_table *nls_io;     /* Charset used for input and display */
        const void *dir_ops;              /* Opaque; default directory operations */
        int dir_per_block;              /* dir entries per block */
        int dir_per_block_bits;              /* log2(dir_per_block) */
        unsigned int vol_id;                /*volume ID*/

        int fatent_shift;
        const struct fatent_operations *fatent_ops;
        struct inode *fat_inode;
        struct inode *fsinfo_inode;

        struct ratelimit_state ratelimit;

        spinlock_t inode_hash_lock;
        struct hlist_head inode_hashtable[FAT_HASH_SIZE];

        spinlock_t dir_hash_lock;
        struct hlist_head dir_hashtable[FAT_HASH_SIZE];

        unsigned int dirty;           /* fs state before mount */
        struct rcu_head rcu;
};

#define FAT_CACHE_VALID        0        /* special case for valid cache */

/*
 * MS-DOS file system inode data in memory
 */
struct msdos_inode_info {
        spinlock_t cache_lru_lock;
        struct list_head cache_lru;
        int nr_caches;
        /* for avoiding the race between fat_free() and fat_get_cluster() */
        unsigned int cache_valid_id;

        /* NOTE: mmu_private is 64bits, so must hold ->i_mutex to access */
        loff_t mmu_private;        /* physically allocated size */

        int i_start;                /* first cluster or 0 */
        int i_logstart;                /* logical first cluster */
        int i_attrs;                /* unused attribute bits */
        loff_t i_pos;                /* on-disk position of directory entry or 0 */
        struct hlist_node i_fat_hash;        /* hash by i_location */
        struct hlist_node i_dir_hash;        /* hash by i_logstart */
        struct rw_semaphore truncate_lock; /* protect bmap against truncate */
        struct timespec64 i_crtime;        /* File creation (birth) time */
        struct inode vfs_inode;
};

struct fat_slot_info {
        loff_t i_pos;                /* on-disk position of directory entry */
        loff_t slot_off;        /* offset for slot or de start */
        int nr_slots;                /* number of slots + 1(de) in filename */
        struct msdos_dir_entry *de;
        struct buffer_head *bh;
};

static inline struct msdos_sb_info *MSDOS_SB(struct super_block *sb)
{
        return sb->s_fs_info;
}

/*
 * Functions that determine the variant of the FAT file system (i.e.,
 * whether this is FAT12, FAT16 or FAT32.
 */
static inline bool is_fat12(const struct msdos_sb_info *sbi)
{
        return sbi->fat_bits == 12;
}

static inline bool is_fat16(const struct msdos_sb_info *sbi)
{
        return sbi->fat_bits == 16;
}

static inline bool is_fat32(const struct msdos_sb_info *sbi)
{
        return sbi->fat_bits == 32;
}

/* Maximum number of clusters */
static inline u32 max_fat(struct super_block *sb)
{
        struct msdos_sb_info *sbi = MSDOS_SB(sb);

        return is_fat32(sbi) ? MAX_FAT32 :
                is_fat16(sbi) ? MAX_FAT16 : MAX_FAT12;
}

static inline struct msdos_inode_info *MSDOS_I(struct inode *inode)
{
        return container_of(inode, struct msdos_inode_info, vfs_inode);
}

/*
 * If ->i_mode can't hold S_IWUGO (i.e. ATTR_RO), we use ->i_attrs to
 * save ATTR_RO instead of ->i_mode.
 *
 * If it's directory and !sbi->options.rodir, ATTR_RO isn't read-only
 * bit, it's just used as flag for app.
 */
static inline int fat_mode_can_hold_ro(struct inode *inode)
{
        struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb);
        umode_t mask;

        if (S_ISDIR(inode->i_mode)) {
                if (!sbi->options.rodir)
                        return 0;
                mask = ~sbi->options.fs_dmask;
        } else
                mask = ~sbi->options.fs_fmask;

        if (!(mask & S_IWUGO))
                return 0;
        return 1;
}

/* Convert attribute bits and a mask to the UNIX mode. */
static inline umode_t fat_make_mode(struct msdos_sb_info *sbi,
                                   u8 attrs, umode_t mode)
{
        if (attrs & ATTR_RO && !((attrs & ATTR_DIR) && !sbi->options.rodir))
                mode &= ~S_IWUGO;

        if (attrs & ATTR_DIR)
                return (mode & ~sbi->options.fs_dmask) | S_IFDIR;
        else
                return (mode & ~sbi->options.fs_fmask) | S_IFREG;
}

/* Return the FAT attribute byte for this inode */
static inline u8 fat_make_attrs(struct inode *inode)
{
        u8 attrs = MSDOS_I(inode)->i_attrs;
        if (S_ISDIR(inode->i_mode))
                attrs |= ATTR_DIR;
        if (fat_mode_can_hold_ro(inode) && !(inode->i_mode & S_IWUGO))
                attrs |= ATTR_RO;
        return attrs;
}

static inline void fat_save_attrs(struct inode *inode, u8 attrs)
{
        if (fat_mode_can_hold_ro(inode))
                MSDOS_I(inode)->i_attrs = attrs & ATTR_UNUSED;
        else
                MSDOS_I(inode)->i_attrs = attrs & (ATTR_UNUSED | ATTR_RO);
}

static inline unsigned char fat_checksum(const __u8 *name)
{
        unsigned char s = name[0];
        s = (s<<7) + (s>>1) + name[1];        s = (s<<7) + (s>>1) + name[2];
        s = (s<<7) + (s>>1) + name[3];        s = (s<<7) + (s>>1) + name[4];
        s = (s<<7) + (s>>1) + name[5];        s = (s<<7) + (s>>1) + name[6];
        s = (s<<7) + (s>>1) + name[7];        s = (s<<7) + (s>>1) + name[8];
        s = (s<<7) + (s>>1) + name[9];        s = (s<<7) + (s>>1) + name[10];
        return s;
}

static inline sector_t fat_clus_to_blknr(struct msdos_sb_info *sbi, int clus)
{
        return ((sector_t)clus - FAT_START_ENT) * sbi->sec_per_clus
                + sbi->data_start;
}

static inline void fat_get_blknr_offset(struct msdos_sb_info *sbi,
                                loff_t i_pos, sector_t *blknr, int *offset)
{
        *blknr = i_pos >> sbi->dir_per_block_bits;
        *offset = i_pos & (sbi->dir_per_block - 1);
}

static inline loff_t fat_i_pos_read(struct msdos_sb_info *sbi,
                                        struct inode *inode)
{
        loff_t i_pos;
#if BITS_PER_LONG == 32
        spin_lock(&sbi->inode_hash_lock);
#endif
        i_pos = MSDOS_I(inode)->i_pos;
#if BITS_PER_LONG == 32
        spin_unlock(&sbi->inode_hash_lock);
#endif
        return i_pos;
}

static inline void fat16_towchar(wchar_t *dst, const __u8 *src, size_t len)
{
#ifdef __BIG_ENDIAN
        while (len--) {
                *dst++ = src[0] | (src[1] << 8);
                src += 2;
        }
#else
        memcpy(dst, src, len * 2);
#endif
}

static inline int fat_get_start(const struct msdos_sb_info *sbi,
                                const struct msdos_dir_entry *de)
{
        int cluster = le16_to_cpu(de->start);
        if (is_fat32(sbi))
                cluster |= (le16_to_cpu(de->starthi) << 16);
        return cluster;
}

static inline void fat_set_start(struct msdos_dir_entry *de, int cluster)
{
        de->start   = cpu_to_le16(cluster);
        de->starthi = cpu_to_le16(cluster >> 16);
}

static inline void fatwchar_to16(__u8 *dst, const wchar_t *src, size_t len)
{
#ifdef __BIG_ENDIAN
        while (len--) {
                dst[0] = *src & 0x00FF;
                dst[1] = (*src & 0xFF00) >> 8;
                dst += 2;
                src++;
        }
#else
        memcpy(dst, src, len * 2);
#endif
}

/* fat/cache.c */
extern void fat_cache_inval_inode(struct inode *inode);
extern int fat_get_cluster(struct inode *inode, int cluster,
                           int *fclus, int *dclus);
extern int fat_get_mapped_cluster(struct inode *inode, sector_t sector,
                                  sector_t last_block,
                                  unsigned long *mapped_blocks, sector_t *bmap);
extern int fat_bmap(struct inode *inode, sector_t sector, sector_t *phys,
                    unsigned long *mapped_blocks, int create, bool from_bmap);

/* fat/dir.c */
extern const struct file_operations fat_dir_operations;
extern int fat_search_long(struct inode *inode, const unsigned char *name,
                           int name_len, struct fat_slot_info *sinfo);
extern int fat_dir_empty(struct inode *dir);
extern int fat_subdirs(struct inode *dir);
extern int fat_scan(struct inode *dir, const unsigned char *name,
                    struct fat_slot_info *sinfo);
extern int fat_scan_logstart(struct inode *dir, int i_logstart,
                             struct fat_slot_info *sinfo);
extern int fat_get_dotdot_entry(struct inode *dir, struct buffer_head **bh,
                                struct msdos_dir_entry **de);
extern int fat_alloc_new_dir(struct inode *dir, struct timespec64 *ts);
extern int fat_add_entries(struct inode *dir, void *slots, int nr_slots,
                           struct fat_slot_info *sinfo);
extern int fat_remove_entries(struct inode *dir, struct fat_slot_info *sinfo);

/* fat/fatent.c */
struct fat_entry {
        int entry;
        union {
                u8 *ent12_p[2];
                __le16 *ent16_p;
                __le32 *ent32_p;
        } u;
        int nr_bhs;
        struct buffer_head *bhs[2];
        struct inode *fat_inode;
};

static inline void fatent_init(struct fat_entry *fatent)
{
        fatent->nr_bhs = 0;
        fatent->entry = 0;
        fatent->u.ent32_p = NULL;
        fatent->bhs[0] = fatent->bhs[1] = NULL;
        fatent->fat_inode = NULL;
}

static inline void fatent_set_entry(struct fat_entry *fatent, int entry)
{
        fatent->entry = entry;
        fatent->u.ent32_p = NULL;
}

static inline void fatent_brelse(struct fat_entry *fatent)
{
        int i;
        fatent->u.ent32_p = NULL;
        for (i = 0; i < fatent->nr_bhs; i++)
                brelse(fatent->bhs[i]);
        fatent->nr_bhs = 0;
        fatent->bhs[0] = fatent->bhs[1] = NULL;
        fatent->fat_inode = NULL;
}

static inline bool fat_valid_entry(struct msdos_sb_info *sbi, int entry)
{
        return FAT_START_ENT <= entry && entry < sbi->max_cluster;
}

extern void fat_ent_access_init(struct super_block *sb);
extern int fat_ent_read(struct inode *inode, struct fat_entry *fatent,
                        int entry);
extern int fat_ent_write(struct inode *inode, struct fat_entry *fatent,
                         int new, int wait);
extern int fat_alloc_clusters(struct inode *inode, int *cluster,
                              int nr_cluster);
extern int fat_free_clusters(struct inode *inode, int cluster);
extern int fat_count_free_clusters(struct super_block *sb);
extern int fat_trim_fs(struct inode *inode, struct fstrim_range *range);

/* fat/file.c */
extern long fat_generic_ioctl(struct file *filp, unsigned int cmd,
                              unsigned long arg);
extern const struct file_operations fat_file_operations;
extern const struct inode_operations fat_file_inode_operations;
extern int fat_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
                       struct iattr *attr);
extern void fat_truncate_blocks(struct inode *inode, loff_t offset);
extern int fat_getattr(struct mnt_idmap *idmap,
                       const struct path *path, struct kstat *stat,
                       u32 request_mask, unsigned int flags);
extern int fat_file_fsync(struct file *file, loff_t start, loff_t end,
                          int datasync);

/* fat/inode.c */
extern int fat_block_truncate_page(struct inode *inode, loff_t from);
extern void fat_attach(struct inode *inode, loff_t i_pos);
extern void fat_detach(struct inode *inode);
extern struct inode *fat_iget(struct super_block *sb, loff_t i_pos);
extern struct inode *fat_build_inode(struct super_block *sb,
                        struct msdos_dir_entry *de, loff_t i_pos);
extern int fat_sync_inode(struct inode *inode);
extern int fat_fill_super(struct super_block *sb, struct fs_context *fc,
                          void (*setup)(struct super_block *));
extern int fat_fill_inode(struct inode *inode, struct msdos_dir_entry *de);

extern int fat_flush_inodes(struct super_block *sb, struct inode *i1,
                            struct inode *i2);

extern const struct fs_parameter_spec fat_param_spec[];
int fat_init_fs_context(struct fs_context *fc, bool is_vfat);
void fat_free_fc(struct fs_context *fc);

int fat_parse_param(struct fs_context *fc, struct fs_parameter *param,
                    bool is_vfat);
int fat_reconfigure(struct fs_context *fc);

static inline unsigned long fat_dir_hash(int logstart)
{
        return hash_32(logstart, FAT_HASH_BITS);
}
extern int fat_add_cluster(struct inode *inode);

/* fat/misc.c */
extern __printf(3, 4) __cold
void __fat_fs_error(struct super_block *sb, int report, const char *fmt, ...);
#define fat_fs_error(sb, fmt, args...)                \
        __fat_fs_error(sb, 1, fmt , ## args)
#define fat_fs_error_ratelimit(sb, fmt, args...) \
        __fat_fs_error(sb, __ratelimit(&MSDOS_SB(sb)->ratelimit), fmt , ## args)

#define FAT_PRINTK_PREFIX "%sFAT-fs (%s): "
#define fat_msg(sb, level, fmt, args...)                                \
do {                                                                        \
        printk_index_subsys_emit(FAT_PRINTK_PREFIX, level, fmt, ##args);\
        _fat_msg(sb, level, fmt, ##args);                                \
} while (0)
__printf(3, 4) __cold
void _fat_msg(struct super_block *sb, const char *level, const char *fmt, ...);
#define fat_msg_ratelimit(sb, level, fmt, args...)        \
        do {        \
                        if (__ratelimit(&MSDOS_SB(sb)->ratelimit))        \
                                fat_msg(sb, level, fmt, ## args);        \
         } while (0)
extern int fat_clusters_flush(struct super_block *sb);
extern int fat_chain_add(struct inode *inode, int new_dclus, int nr_cluster);
extern void fat_time_fat2unix(struct msdos_sb_info *sbi, struct timespec64 *ts,
                              __le16 __time, __le16 __date, u8 time_cs);
extern void fat_time_unix2fat(struct msdos_sb_info *sbi, struct timespec64 *ts,
                              __le16 *time, __le16 *date, u8 *time_cs);
extern struct timespec64 fat_truncate_atime(const struct msdos_sb_info *sbi,
                                            const struct timespec64 *ts);
extern struct timespec64 fat_truncate_mtime(const struct msdos_sb_info *sbi,
                                            const struct timespec64 *ts);
extern int fat_truncate_time(struct inode *inode, struct timespec64 *now,
                             int flags);
extern int fat_update_time(struct inode *inode, int flags);
extern int fat_sync_bhs(struct buffer_head **bhs, int nr_bhs);

int fat_cache_init(void);
void fat_cache_destroy(void);

/* fat/nfs.c */
extern const struct export_operations fat_export_ops;
extern const struct export_operations fat_export_ops_nostale;

/* helper for printk */
typedef unsigned long long        llu;

#endif /* !_FAT_H */






















































































































































































































































    2 







































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2018 HUAWEI, Inc.
 *             https://www.huawei.com/
 * Copyright (C) 2024 Alibaba Cloud
 */
#include "internal.h"

struct z_erofs_gbuf {
        spinlock_t lock;
        void *ptr;
        struct page **pages;
        unsigned int nrpages;
};

static struct z_erofs_gbuf *z_erofs_gbufpool, *z_erofs_rsvbuf;
static unsigned int z_erofs_gbuf_count, z_erofs_gbuf_nrpages,
                z_erofs_rsv_nrpages;

module_param_named(global_buffers, z_erofs_gbuf_count, uint, 0444);
module_param_named(reserved_pages, z_erofs_rsv_nrpages, uint, 0444);

atomic_long_t erofs_global_shrink_cnt;        /* for all mounted instances */

/* protects `erofs_sb_list_lock` and the mounted `erofs_sb_list` */
static DEFINE_SPINLOCK(erofs_sb_list_lock);
static LIST_HEAD(erofs_sb_list);
static unsigned int shrinker_run_no;
static struct shrinker *erofs_shrinker_info;

static unsigned int z_erofs_gbuf_id(void)
{
        return raw_smp_processor_id() % z_erofs_gbuf_count;
}

void *z_erofs_get_gbuf(unsigned int requiredpages)
        __acquires(gbuf->lock)
{
        struct z_erofs_gbuf *gbuf;

        migrate_disable();
        gbuf = &z_erofs_gbufpool[z_erofs_gbuf_id()];
        spin_lock(&gbuf->lock);
        /* check if the buffer is too small */
        if (requiredpages > gbuf->nrpages) {
                spin_unlock(&gbuf->lock);
                migrate_enable();
                /* (for sparse checker) pretend gbuf->lock is still taken */
                __acquire(gbuf->lock);
                return NULL;
        }
        return gbuf->ptr;
}

void z_erofs_put_gbuf(void *ptr) __releases(gbuf->lock)
{
        struct z_erofs_gbuf *gbuf;

        gbuf = &z_erofs_gbufpool[z_erofs_gbuf_id()];
        DBG_BUGON(gbuf->ptr != ptr);
        spin_unlock(&gbuf->lock);
        migrate_enable();
}

int z_erofs_gbuf_growsize(unsigned int nrpages)
{
        static DEFINE_MUTEX(gbuf_resize_mutex);
        struct page **tmp_pages = NULL;
        struct z_erofs_gbuf *gbuf;
        void *ptr, *old_ptr;
        int last, i, j;

        mutex_lock(&gbuf_resize_mutex);
        /* avoid shrinking gbufs, since no idea how many fses rely on */
        if (nrpages <= z_erofs_gbuf_nrpages) {
                mutex_unlock(&gbuf_resize_mutex);
                return 0;
        }

        for (i = 0; i < z_erofs_gbuf_count; ++i) {
                gbuf = &z_erofs_gbufpool[i];
                tmp_pages = kcalloc(nrpages, sizeof(*tmp_pages), GFP_KERNEL);
                if (!tmp_pages)
                        goto out;

                for (j = 0; j < gbuf->nrpages; ++j)
                        tmp_pages[j] = gbuf->pages[j];
                do {
                        last = j;
                        j = alloc_pages_bulk(GFP_KERNEL, nrpages,
                                             tmp_pages);
                        if (last == j)
                                goto out;
                } while (j != nrpages);

                ptr = vmap(tmp_pages, nrpages, VM_MAP, PAGE_KERNEL);
                if (!ptr)
                        goto out;

                spin_lock(&gbuf->lock);
                kfree(gbuf->pages);
                gbuf->pages = tmp_pages;
                old_ptr = gbuf->ptr;
                gbuf->ptr = ptr;
                gbuf->nrpages = nrpages;
                spin_unlock(&gbuf->lock);
                if (old_ptr)
                        vunmap(old_ptr);
        }
        z_erofs_gbuf_nrpages = nrpages;
out:
        if (i < z_erofs_gbuf_count && tmp_pages) {
                for (j = 0; j < nrpages; ++j)
                        if (tmp_pages[j] && (j >= gbuf->nrpages ||
                                             tmp_pages[j] != gbuf->pages[j]))
                                __free_page(tmp_pages[j]);
                kfree(tmp_pages);
        }
        mutex_unlock(&gbuf_resize_mutex);
        return i < z_erofs_gbuf_count ? -ENOMEM : 0;
}

int __init z_erofs_gbuf_init(void)
{
        unsigned int i, total = num_possible_cpus();

        if (z_erofs_gbuf_count)
                total = min(z_erofs_gbuf_count, total);
        z_erofs_gbuf_count = total;

        /* The last (special) global buffer is the reserved buffer */
        total += !!z_erofs_rsv_nrpages;

        z_erofs_gbufpool = kcalloc(total, sizeof(*z_erofs_gbufpool),
                                   GFP_KERNEL);
        if (!z_erofs_gbufpool)
                return -ENOMEM;

        if (z_erofs_rsv_nrpages) {
                z_erofs_rsvbuf = &z_erofs_gbufpool[total - 1];
                z_erofs_rsvbuf->pages = kcalloc(z_erofs_rsv_nrpages,
                                sizeof(*z_erofs_rsvbuf->pages), GFP_KERNEL);
                if (!z_erofs_rsvbuf->pages) {
                        z_erofs_rsvbuf = NULL;
                        z_erofs_rsv_nrpages = 0;
                }
        }
        for (i = 0; i < total; ++i)
                spin_lock_init(&z_erofs_gbufpool[i].lock);
        return 0;
}

void z_erofs_gbuf_exit(void)
{
        int i, j;

        for (i = 0; i < z_erofs_gbuf_count + (!!z_erofs_rsvbuf); ++i) {
                struct z_erofs_gbuf *gbuf = &z_erofs_gbufpool[i];

                if (gbuf->ptr) {
                        vunmap(gbuf->ptr);
                        gbuf->ptr = NULL;
                }

                if (!gbuf->pages)
                        continue;

                for (j = 0; j < gbuf->nrpages; ++j)
                        if (gbuf->pages[j])
                                put_page(gbuf->pages[j]);
                kfree(gbuf->pages);
                gbuf->pages = NULL;
        }
        kfree(z_erofs_gbufpool);
}

struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp, bool tryrsv)
{
        struct page *page = *pagepool;

        if (page) {
                *pagepool = (struct page *)page_private(page);
        } else if (tryrsv && z_erofs_rsvbuf && z_erofs_rsvbuf->nrpages) {
                spin_lock(&z_erofs_rsvbuf->lock);
                if (z_erofs_rsvbuf->nrpages)
                        page = z_erofs_rsvbuf->pages[--z_erofs_rsvbuf->nrpages];
                spin_unlock(&z_erofs_rsvbuf->lock);
        }
        if (!page)
                page = alloc_page(gfp);
        DBG_BUGON(page && page_ref_count(page) != 1);
        return page;
}

void erofs_release_pages(struct page **pagepool)
{
        while (*pagepool) {
                struct page *page = *pagepool;

                *pagepool = (struct page *)page_private(page);
                /* try to fill reserved global pool first */
                if (z_erofs_rsvbuf && z_erofs_rsvbuf->nrpages <
                                z_erofs_rsv_nrpages) {
                        spin_lock(&z_erofs_rsvbuf->lock);
                        if (z_erofs_rsvbuf->nrpages < z_erofs_rsv_nrpages) {
                                z_erofs_rsvbuf->pages[z_erofs_rsvbuf->nrpages++]
                                                = page;
                                spin_unlock(&z_erofs_rsvbuf->lock);
                                continue;
                        }
                        spin_unlock(&z_erofs_rsvbuf->lock);
                }
                put_page(page);
        }
}

void erofs_shrinker_register(struct super_block *sb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);

        mutex_init(&sbi->umount_mutex);

        spin_lock(&erofs_sb_list_lock);
        list_add(&sbi->list, &erofs_sb_list);
        spin_unlock(&erofs_sb_list_lock);
}

void erofs_shrinker_unregister(struct super_block *sb)
{
        struct erofs_sb_info *const sbi = EROFS_SB(sb);

        mutex_lock(&sbi->umount_mutex);
        while (!xa_empty(&sbi->managed_pslots)) {
                z_erofs_shrink_scan(sbi, ~0UL);
                cond_resched();
        }
        spin_lock(&erofs_sb_list_lock);
        list_del(&sbi->list);
        spin_unlock(&erofs_sb_list_lock);
        mutex_unlock(&sbi->umount_mutex);
}

static unsigned long erofs_shrink_count(struct shrinker *shrink,
                                        struct shrink_control *sc)
{
        return atomic_long_read(&erofs_global_shrink_cnt) ?: SHRINK_EMPTY;
}

static unsigned long erofs_shrink_scan(struct shrinker *shrink,
                                       struct shrink_control *sc)
{
        struct erofs_sb_info *sbi;
        struct list_head *p;

        unsigned long nr = sc->nr_to_scan;
        unsigned int run_no;
        unsigned long freed = 0;

        spin_lock(&erofs_sb_list_lock);
        do {
                run_no = ++shrinker_run_no;
        } while (run_no == 0);

        /* Iterate over all mounted superblocks and try to shrink them */
        p = erofs_sb_list.next;
        while (p != &erofs_sb_list) {
                sbi = list_entry(p, struct erofs_sb_info, list);

                /*
                 * We move the ones we do to the end of the list, so we stop
                 * when we see one we have already done.
                 */
                if (sbi->shrinker_run_no == run_no)
                        break;

                if (!mutex_trylock(&sbi->umount_mutex)) {
                        p = p->next;
                        continue;
                }

                spin_unlock(&erofs_sb_list_lock);
                sbi->shrinker_run_no = run_no;
                freed += z_erofs_shrink_scan(sbi, nr - freed);
                spin_lock(&erofs_sb_list_lock);
                /* Get the next list element before we move this one */
                p = p->next;

                /*
                 * Move this one to the end of the list to provide some
                 * fairness.
                 */
                list_move_tail(&sbi->list, &erofs_sb_list);
                mutex_unlock(&sbi->umount_mutex);

                if (freed >= nr)
                        break;
        }
        spin_unlock(&erofs_sb_list_lock);
        return freed;
}

int __init erofs_init_shrinker(void)
{
        erofs_shrinker_info = shrinker_alloc(0, "erofs-shrinker");
        if (!erofs_shrinker_info)
                return -ENOMEM;

        erofs_shrinker_info->count_objects = erofs_shrink_count;
        erofs_shrinker_info->scan_objects = erofs_shrink_scan;
        shrinker_register(erofs_shrinker_info);
        return 0;
}

void erofs_exit_shrinker(void)
{
        shrinker_free(erofs_shrinker_info);
}




















































































































































































































































































    5 
    4 


    5 
    5 

    5 

    2 

    2 





    1 

    5 
























































































































    2 




    1 




































    2 





    1 






















































































    1 





    1 




















    5 

































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
/*
 * Equalizer Load-balancer for serial network interfaces.
 *
 * (c) Copyright 1995 Simon "Guru Aleph-Null" Janes
 * NCM: Network and Communications Management, Inc.
 *
 * (c) Copyright 2002 David S. Miller (davem@redhat.com)
 *
 *        This software may be used and distributed according to the terms
 *        of the GNU General Public License, incorporated herein by reference.
 *
 * The author may be reached as simon@ncm.com, or C/O
 *    NCM
 *    Attn: Simon Janes
 *    6803 Whittier Ave
 *    McLean VA 22101
 *    Phone: 1-703-847-0040 ext 103
 */

/*
 * Sources:
 *   skeleton.c by Donald Becker.
 * Inspirations:
 *   The Harried and Overworked Alan Cox
 * Conspiracies:
 *   The Alan Cox and Mike McLagan plot to get someone else to do the code,
 *   which turned out to be me.
 */

/*
 * $Log: eql.c,v $
 * Revision 1.2  1996/04/11 17:51:52  guru
 * Added one-line eql_remove_slave patch.
 *
 * Revision 1.1  1996/04/11 17:44:17  guru
 * Initial revision
 *
 * Revision 3.13  1996/01/21  15:17:18  alan
 * tx_queue_len changes.
 * reformatted.
 *
 * Revision 3.12  1995/03/22  21:07:51  anarchy
 * Added capable() checks on configuration.
 * Moved header file.
 *
 * Revision 3.11  1995/01/19  23:14:31  guru
 *                       slave_load = (ULONG_MAX - (ULONG_MAX / 2)) -
 *                         (priority_Bps) + bytes_queued * 8;
 *
 * Revision 3.10  1995/01/19  23:07:53  guru
 * back to
 *                       slave_load = (ULONG_MAX - (ULONG_MAX / 2)) -
 *                         (priority_Bps) + bytes_queued;
 *
 * Revision 3.9  1995/01/19  22:38:20  guru
 *                       slave_load = (ULONG_MAX - (ULONG_MAX / 2)) -
 *                         (priority_Bps) + bytes_queued * 4;
 *
 * Revision 3.8  1995/01/19  22:30:55  guru
 *       slave_load = (ULONG_MAX - (ULONG_MAX / 2)) -
 *                         (priority_Bps) + bytes_queued * 2;
 *
 * Revision 3.7  1995/01/19  21:52:35  guru
 * printk's trimmed out.
 *
 * Revision 3.6  1995/01/19  21:49:56  guru
 * This is working pretty well. I gained 1 K/s in speed.. now it's just
 * robustness and printk's to be diked out.
 *
 * Revision 3.5  1995/01/18  22:29:59  guru
 * still crashes the kernel when the lock_wait thing is woken up.
 *
 * Revision 3.4  1995/01/18  21:59:47  guru
 * Broken set-bit locking snapshot
 *
 * Revision 3.3  1995/01/17  22:09:18  guru
 * infinite sleep in a lock somewhere..
 *
 * Revision 3.2  1995/01/15  16:46:06  guru
 * Log trimmed of non-pertinent 1.x branch messages
 *
 * Revision 3.1  1995/01/15  14:41:45  guru
 * New Scheduler and timer stuff...
 *
 * Revision 1.15  1995/01/15  14:29:02  guru
 * Will make 1.14 (now 1.15) the 3.0 branch, and the 1.12 the 2.0 branch, the one
 * with the dumber scheduler
 *
 * Revision 1.14  1995/01/15  02:37:08  guru
 * shock.. the kept-new-versions could have zonked working
 * stuff.. shudder
 *
 * Revision 1.13  1995/01/15  02:36:31  guru
 * big changes
 *
 *         scheduler was torn out and replaced with something smarter
 *
 *         global names not prefixed with eql_ were renamed to protect
 *         against namespace collisions
 *
 *         a few more abstract interfaces were added to facilitate any
 *         potential change of datastructure.  the driver is still using
 *         a linked list of slaves.  going to a heap would be a bit of
 *         an overkill.
 *
 *         this compiles fine with no warnings.
 *
 *         the locking mechanism and timer stuff must be written however,
 *         this version will not work otherwise
 *
 * Sorry, I had to rewrite most of this for 2.5.x -DaveM
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/compat.h>
#include <linux/capability.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/netdevice.h>
#include <net/net_namespace.h>

#include <linux/if.h>
#include <linux/if_arp.h>
#include <linux/if_eql.h>
#include <linux/pkt_sched.h>

#include <linux/uaccess.h>

static int eql_open(struct net_device *dev);
static int eql_close(struct net_device *dev);
static int eql_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
                              void __user *data, int cmd);
static netdev_tx_t eql_slave_xmit(struct sk_buff *skb, struct net_device *dev);

#define eql_is_slave(dev)        ((dev->flags & IFF_SLAVE) == IFF_SLAVE)
#define eql_is_master(dev)        ((dev->flags & IFF_MASTER) == IFF_MASTER)

static void eql_kill_one_slave(slave_queue_t *queue, slave_t *slave);

static void eql_timer(struct timer_list *t)
{
        equalizer_t *eql = from_timer(eql, t, timer);
        struct list_head *this, *tmp, *head;

        spin_lock(&eql->queue.lock);
        head = &eql->queue.all_slaves;
        list_for_each_safe(this, tmp, head) {
                slave_t *slave = list_entry(this, slave_t, list);

                if ((slave->dev->flags & IFF_UP) == IFF_UP) {
                        slave->bytes_queued -= slave->priority_Bps;
                        if (slave->bytes_queued < 0)
                                slave->bytes_queued = 0;
                } else {
                        eql_kill_one_slave(&eql->queue, slave);
                }

        }
        spin_unlock(&eql->queue.lock);

        eql->timer.expires = jiffies + EQL_DEFAULT_RESCHED_IVAL;
        add_timer(&eql->timer);
}

static const char version[] __initconst =
        "Equalizer2002: Simon Janes (simon@ncm.com) and David S. Miller (davem@redhat.com)";

static const struct net_device_ops eql_netdev_ops = {
        .ndo_open        = eql_open,
        .ndo_stop        = eql_close,
        .ndo_siocdevprivate = eql_siocdevprivate,
        .ndo_start_xmit        = eql_slave_xmit,
};

static void __init eql_setup(struct net_device *dev)
{
        equalizer_t *eql = netdev_priv(dev);

        timer_setup(&eql->timer, eql_timer, 0);
        eql->timer.expires          = jiffies + EQL_DEFAULT_RESCHED_IVAL;

        spin_lock_init(&eql->queue.lock);
        INIT_LIST_HEAD(&eql->queue.all_slaves);
        eql->queue.master_dev        = dev;

        dev->netdev_ops                = &eql_netdev_ops;

        /*
         *        Now we undo some of the things that eth_setup does
         *         that we don't like
         */

        dev->mtu                = EQL_DEFAULT_MTU;        /* set to 576 in if_eql.h */
        dev->flags              = IFF_MASTER;

        dev->type               = ARPHRD_SLIP;
        dev->tx_queue_len         = 5;                /* Hands them off fast */
        netif_keep_dst(dev);
}

static int eql_open(struct net_device *dev)
{
        equalizer_t *eql = netdev_priv(dev);

        /* XXX We should force this off automatically for the user. */
        netdev_info(dev,
                    "remember to turn off Van-Jacobson compression on your slave devices\n");

        BUG_ON(!list_empty(&eql->queue.all_slaves));

        eql->min_slaves = 1;
        eql->max_slaves = EQL_DEFAULT_MAX_SLAVES; /* 4 usually... */

        add_timer(&eql->timer);

        return 0;
}

static void eql_kill_one_slave(slave_queue_t *queue, slave_t *slave)
{
        list_del(&slave->list);
        queue->num_slaves--;
        slave->dev->flags &= ~IFF_SLAVE;
        netdev_put(slave->dev, &slave->dev_tracker);
        kfree(slave);
}

static void eql_kill_slave_queue(slave_queue_t *queue)
{
        struct list_head *head, *tmp, *this;

        spin_lock_bh(&queue->lock);

        head = &queue->all_slaves;
        list_for_each_safe(this, tmp, head) {
                slave_t *s = list_entry(this, slave_t, list);

                eql_kill_one_slave(queue, s);
        }

        spin_unlock_bh(&queue->lock);
}

static int eql_close(struct net_device *dev)
{
        equalizer_t *eql = netdev_priv(dev);

        /*
         *        The timer has to be stopped first before we start hacking away
         *        at the data structure it scans every so often...
         */

        del_timer_sync(&eql->timer);

        eql_kill_slave_queue(&eql->queue);

        return 0;
}

static int eql_enslave(struct net_device *dev,  slaving_request_t __user *srq);
static int eql_emancipate(struct net_device *dev, slaving_request_t __user *srq);

static int eql_g_slave_cfg(struct net_device *dev, slave_config_t __user *sc);
static int eql_s_slave_cfg(struct net_device *dev, slave_config_t __user *sc);

static int eql_g_master_cfg(struct net_device *dev, master_config_t __user *mc);
static int eql_s_master_cfg(struct net_device *dev, master_config_t __user *mc);

static int eql_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
                              void __user *data, int cmd)
{
        if (cmd != EQL_GETMASTRCFG && cmd != EQL_GETSLAVECFG &&
            !capable(CAP_NET_ADMIN))
                  return -EPERM;

        if (in_compat_syscall()) /* to be implemented */
                return -EOPNOTSUPP;

        switch (cmd) {
                case EQL_ENSLAVE:
                        return eql_enslave(dev, data);
                case EQL_EMANCIPATE:
                        return eql_emancipate(dev, data);
                case EQL_GETSLAVECFG:
                        return eql_g_slave_cfg(dev, data);
                case EQL_SETSLAVECFG:
                        return eql_s_slave_cfg(dev, data);
                case EQL_GETMASTRCFG:
                        return eql_g_master_cfg(dev, data);
                case EQL_SETMASTRCFG:
                        return eql_s_master_cfg(dev, data);
                default:
                        return -EOPNOTSUPP;
        }
}

/* queue->lock must be held */
static slave_t *__eql_schedule_slaves(slave_queue_t *queue)
{
        unsigned long best_load = ~0UL;
        struct list_head *this, *tmp, *head;
        slave_t *best_slave;

        best_slave = NULL;

        /* Make a pass to set the best slave. */
        head = &queue->all_slaves;
        list_for_each_safe(this, tmp, head) {
                slave_t *slave = list_entry(this, slave_t, list);
                unsigned long slave_load, bytes_queued, priority_Bps;

                /* Go through the slave list once, updating best_slave
                 * whenever a new best_load is found.
                 */
                bytes_queued = slave->bytes_queued;
                priority_Bps = slave->priority_Bps;
                if ((slave->dev->flags & IFF_UP) == IFF_UP) {
                        slave_load = (~0UL - (~0UL / 2)) -
                                (priority_Bps) + bytes_queued * 8;

                        if (slave_load < best_load) {
                                best_load = slave_load;
                                best_slave = slave;
                        }
                } else {
                        /* We found a dead slave, kill it. */
                        eql_kill_one_slave(queue, slave);
                }
        }
        return best_slave;
}

static netdev_tx_t eql_slave_xmit(struct sk_buff *skb, struct net_device *dev)
{
        equalizer_t *eql = netdev_priv(dev);
        slave_t *slave;

        spin_lock(&eql->queue.lock);

        slave = __eql_schedule_slaves(&eql->queue);
        if (slave) {
                struct net_device *slave_dev = slave->dev;

                skb->dev = slave_dev;
                skb->priority = TC_PRIO_FILLER;
                slave->bytes_queued += skb->len;
                dev_queue_xmit(skb);
                dev->stats.tx_packets++;
        } else {
                dev->stats.tx_dropped++;
                dev_kfree_skb(skb);
        }

        spin_unlock(&eql->queue.lock);

        return NETDEV_TX_OK;
}

/*
 *        Private ioctl functions
 */

/* queue->lock must be held */
static slave_t *__eql_find_slave_dev(slave_queue_t *queue, struct net_device *dev)
{
        struct list_head *this, *head;

        head = &queue->all_slaves;
        list_for_each(this, head) {
                slave_t *slave = list_entry(this, slave_t, list);

                if (slave->dev == dev)
                        return slave;
        }

        return NULL;
}

static inline int eql_is_full(slave_queue_t *queue)
{
        equalizer_t *eql = netdev_priv(queue->master_dev);

        if (queue->num_slaves >= eql->max_slaves)
                return 1;
        return 0;
}

/* queue->lock must be held */
static int __eql_insert_slave(slave_queue_t *queue, slave_t *slave)
{
        if (!eql_is_full(queue)) {
                slave_t *duplicate_slave = NULL;

                duplicate_slave = __eql_find_slave_dev(queue, slave->dev);
                if (duplicate_slave)
                        eql_kill_one_slave(queue, duplicate_slave);

                netdev_hold(slave->dev, &slave->dev_tracker, GFP_ATOMIC);
                list_add(&slave->list, &queue->all_slaves);
                queue->num_slaves++;
                slave->dev->flags |= IFF_SLAVE;

                return 0;
        }

        return -ENOSPC;
}

static int eql_enslave(struct net_device *master_dev, slaving_request_t __user *srqp)
{
        struct net_device *slave_dev;
        slaving_request_t srq;

        if (copy_from_user(&srq, srqp, sizeof (slaving_request_t)))
                return -EFAULT;

        slave_dev = __dev_get_by_name(&init_net, srq.slave_name);
        if (!slave_dev)
                return -ENODEV;

        if ((master_dev->flags & IFF_UP) == IFF_UP) {
                /* slave is not a master & not already a slave: */
                if (!eql_is_master(slave_dev) && !eql_is_slave(slave_dev)) {
                        slave_t *s = kzalloc(sizeof(*s), GFP_KERNEL);
                        equalizer_t *eql = netdev_priv(master_dev);
                        int ret;

                        if (!s)
                                return -ENOMEM;

                        s->dev = slave_dev;
                        s->priority = srq.priority;
                        s->priority_bps = srq.priority;
                        s->priority_Bps = srq.priority / 8;

                        spin_lock_bh(&eql->queue.lock);
                        ret = __eql_insert_slave(&eql->queue, s);
                        if (ret)
                                kfree(s);

                        spin_unlock_bh(&eql->queue.lock);

                        return ret;
                }
        }

        return -EINVAL;
}

static int eql_emancipate(struct net_device *master_dev, slaving_request_t __user *srqp)
{
        equalizer_t *eql = netdev_priv(master_dev);
        struct net_device *slave_dev;
        slaving_request_t srq;
        int ret;

        if (copy_from_user(&srq, srqp, sizeof (slaving_request_t)))
                return -EFAULT;

        slave_dev = __dev_get_by_name(&init_net, srq.slave_name);
        if (!slave_dev)
                return -ENODEV;

        ret = -EINVAL;
        spin_lock_bh(&eql->queue.lock);
        if (eql_is_slave(slave_dev)) {
                slave_t *slave = __eql_find_slave_dev(&eql->queue, slave_dev);
                if (slave) {
                        eql_kill_one_slave(&eql->queue, slave);
                        ret = 0;
                }
        }
        spin_unlock_bh(&eql->queue.lock);

        return ret;
}

static int eql_g_slave_cfg(struct net_device *dev, slave_config_t __user *scp)
{
        equalizer_t *eql = netdev_priv(dev);
        slave_t *slave;
        struct net_device *slave_dev;
        slave_config_t sc;
        int ret;

        if (copy_from_user(&sc, scp, sizeof (slave_config_t)))
                return -EFAULT;

        slave_dev = __dev_get_by_name(&init_net, sc.slave_name);
        if (!slave_dev)
                return -ENODEV;

        ret = -EINVAL;

        spin_lock_bh(&eql->queue.lock);
        if (eql_is_slave(slave_dev)) {
                slave = __eql_find_slave_dev(&eql->queue, slave_dev);
                if (slave) {
                        sc.priority = slave->priority;
                        ret = 0;
                }
        }
        spin_unlock_bh(&eql->queue.lock);

        if (!ret && copy_to_user(scp, &sc, sizeof (slave_config_t)))
                ret = -EFAULT;

        return ret;
}

static int eql_s_slave_cfg(struct net_device *dev, slave_config_t __user *scp)
{
        slave_t *slave;
        equalizer_t *eql;
        struct net_device *slave_dev;
        slave_config_t sc;
        int ret;

        if (copy_from_user(&sc, scp, sizeof (slave_config_t)))
                return -EFAULT;

        slave_dev = __dev_get_by_name(&init_net, sc.slave_name);
        if (!slave_dev)
                return -ENODEV;

        ret = -EINVAL;

        eql = netdev_priv(dev);
        spin_lock_bh(&eql->queue.lock);
        if (eql_is_slave(slave_dev)) {
                slave = __eql_find_slave_dev(&eql->queue, slave_dev);
                if (slave) {
                        slave->priority = sc.priority;
                        slave->priority_bps = sc.priority;
                        slave->priority_Bps = sc.priority / 8;
                        ret = 0;
                }
        }
        spin_unlock_bh(&eql->queue.lock);

        return ret;
}

static int eql_g_master_cfg(struct net_device *dev, master_config_t __user *mcp)
{
        equalizer_t *eql;
        master_config_t mc;

        memset(&mc, 0, sizeof(master_config_t));

        if (eql_is_master(dev)) {
                eql = netdev_priv(dev);
                mc.max_slaves = eql->max_slaves;
                mc.min_slaves = eql->min_slaves;
                if (copy_to_user(mcp, &mc, sizeof (master_config_t)))
                        return -EFAULT;
                return 0;
        }
        return -EINVAL;
}

static int eql_s_master_cfg(struct net_device *dev, master_config_t __user *mcp)
{
        equalizer_t *eql;
        master_config_t mc;

        if (copy_from_user(&mc, mcp, sizeof (master_config_t)))
                return -EFAULT;

        if (eql_is_master(dev)) {
                eql = netdev_priv(dev);
                eql->max_slaves = mc.max_slaves;
                eql->min_slaves = mc.min_slaves;
                return 0;
        }
        return -EINVAL;
}

static struct net_device *dev_eql;

static int __init eql_init_module(void)
{
        int err;

        pr_info("%s\n", version);

        dev_eql = alloc_netdev(sizeof(equalizer_t), "eql", NET_NAME_UNKNOWN,
                               eql_setup);
        if (!dev_eql)
                return -ENOMEM;

        err = register_netdev(dev_eql);
        if (err)
                free_netdev(dev_eql);
        return err;
}

static void __exit eql_cleanup_module(void)
{
        unregister_netdev(dev_eql);
        free_netdev(dev_eql);
}

module_init(eql_init_module);
module_exit(eql_cleanup_module);
MODULE_DESCRIPTION("Equalizer Load-balancer for serial network interfaces");
MODULE_LICENSE("GPL");

















































































































































































































































































































































































































































  150 




   83 


















   28 
   21 






























   61 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Block data types and constants.  Directly include this file only to
 * break include dependency loop.
 */
#ifndef __LINUX_BLK_TYPES_H
#define __LINUX_BLK_TYPES_H

#include <linux/types.h>
#include <linux/bvec.h>
#include <linux/device.h>
#include <linux/ktime.h>
#include <linux/rw_hint.h>

struct bio_set;
struct bio;
struct bio_integrity_payload;
struct page;
struct io_context;
struct cgroup_subsys_state;
typedef void (bio_end_io_t) (struct bio *);
struct bio_crypt_ctx;

/*
 * The basic unit of block I/O is a sector. It is used in a number of contexts
 * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9
 * bytes. Variables of type sector_t represent an offset or size that is a
 * multiple of 512 bytes. Hence these two constants.
 */
#ifndef SECTOR_SHIFT
#define SECTOR_SHIFT 9
#endif
#ifndef SECTOR_SIZE
#define SECTOR_SIZE (1 << SECTOR_SHIFT)
#endif

#define PAGE_SECTORS_SHIFT        (PAGE_SHIFT - SECTOR_SHIFT)
#define PAGE_SECTORS                (1 << PAGE_SECTORS_SHIFT)
#define SECTOR_MASK                (PAGE_SECTORS - 1)

struct block_device {
        sector_t                bd_start_sect;
        sector_t                bd_nr_sectors;
        struct gendisk *        bd_disk;
        struct request_queue *        bd_queue;
        struct disk_stats __percpu *bd_stats;
        unsigned long                bd_stamp;
        atomic_t                __bd_flags;        // partition number + flags
#define BD_PARTNO                255        // lower 8 bits; assign-once
#define BD_READ_ONLY                (1u<<8) // read-only policy
#define BD_WRITE_HOLDER                (1u<<9)
#define BD_HAS_SUBMIT_BIO        (1u<<10)
#define BD_RO_WARNED                (1u<<11)
#ifdef CONFIG_FAIL_MAKE_REQUEST
#define BD_MAKE_IT_FAIL                (1u<<12)
#endif
        dev_t                        bd_dev;
        struct address_space        *bd_mapping;        /* page cache */

        atomic_t                bd_openers;
        spinlock_t                bd_size_lock; /* for bd_inode->i_size updates */
        void *                        bd_claiming;
        void *                        bd_holder;
        const struct blk_holder_ops *bd_holder_ops;
        struct mutex                bd_holder_lock;
        int                        bd_holders;
        struct kobject                *bd_holder_dir;

        atomic_t                bd_fsfreeze_count; /* number of freeze requests */
        struct mutex                bd_fsfreeze_mutex; /* serialize freeze/thaw */

        struct partition_meta_info *bd_meta_info;
        int                        bd_writers;
#ifdef CONFIG_SECURITY
        void                        *bd_security;
#endif
        /*
         * keep this out-of-line as it's both big and not needed in the fast
         * path
         */
        struct device                bd_device;
} __randomize_layout;

#define bdev_whole(_bdev) \
        ((_bdev)->bd_disk->part0)

#define dev_to_bdev(device) \
        container_of((device), struct block_device, bd_device)

#define bdev_kobj(_bdev) \
        (&((_bdev)->bd_device.kobj))

/*
 * Block error status values.  See block/blk-core:blk_errors for the details.
 */
typedef u8 __bitwise blk_status_t;
typedef u16 blk_short_t;
#define        BLK_STS_OK 0
#define BLK_STS_NOTSUPP                ((__force blk_status_t)1)
#define BLK_STS_TIMEOUT                ((__force blk_status_t)2)
#define BLK_STS_NOSPC                ((__force blk_status_t)3)
#define BLK_STS_TRANSPORT        ((__force blk_status_t)4)
#define BLK_STS_TARGET                ((__force blk_status_t)5)
#define BLK_STS_RESV_CONFLICT        ((__force blk_status_t)6)
#define BLK_STS_MEDIUM                ((__force blk_status_t)7)
#define BLK_STS_PROTECTION        ((__force blk_status_t)8)
#define BLK_STS_RESOURCE        ((__force blk_status_t)9)
#define BLK_STS_IOERR                ((__force blk_status_t)10)

/* hack for device mapper, don't use elsewhere: */
#define BLK_STS_DM_REQUEUE    ((__force blk_status_t)11)

/*
 * BLK_STS_AGAIN should only be returned if RQF_NOWAIT is set
 * and the bio would block (cf bio_wouldblock_error())
 */
#define BLK_STS_AGAIN                ((__force blk_status_t)12)

/*
 * BLK_STS_DEV_RESOURCE is returned from the driver to the block layer if
 * device related resources are unavailable, but the driver can guarantee
 * that the queue will be rerun in the future once resources become
 * available again. This is typically the case for device specific
 * resources that are consumed for IO. If the driver fails allocating these
 * resources, we know that inflight (or pending) IO will free these
 * resource upon completion.
 *
 * This is different from BLK_STS_RESOURCE in that it explicitly references
 * a device specific resource. For resources of wider scope, allocation
 * failure can happen without having pending IO. This means that we can't
 * rely on request completions freeing these resources, as IO may not be in
 * flight. Examples of that are kernel memory allocations, DMA mappings, or
 * any other system wide resources.
 */
#define BLK_STS_DEV_RESOURCE        ((__force blk_status_t)13)

/*
 * BLK_STS_ZONE_OPEN_RESOURCE is returned from the driver in the completion
 * path if the device returns a status indicating that too many zone resources
 * are currently open. The same command should be successful if resubmitted
 * after the number of open zones decreases below the device's limits, which is
 * reported in the request_queue's max_open_zones.
 */
#define BLK_STS_ZONE_OPEN_RESOURCE        ((__force blk_status_t)14)

/*
 * BLK_STS_ZONE_ACTIVE_RESOURCE is returned from the driver in the completion
 * path if the device returns a status indicating that too many zone resources
 * are currently active. The same command should be successful if resubmitted
 * after the number of active zones decreases below the device's limits, which
 * is reported in the request_queue's max_active_zones.
 */
#define BLK_STS_ZONE_ACTIVE_RESOURCE        ((__force blk_status_t)15)

/*
 * BLK_STS_OFFLINE is returned from the driver when the target device is offline
 * or is being taken offline. This could help differentiate the case where a
 * device is intentionally being shut down from a real I/O error.
 */
#define BLK_STS_OFFLINE                ((__force blk_status_t)16)

/*
 * BLK_STS_DURATION_LIMIT is returned from the driver when the target device
 * aborted the command because it exceeded one of its Command Duration Limits.
 */
#define BLK_STS_DURATION_LIMIT        ((__force blk_status_t)17)

/*
 * Invalid size or alignment.
 */
#define BLK_STS_INVAL        ((__force blk_status_t)19)

/**
 * blk_path_error - returns true if error may be path related
 * @error: status the request was completed with
 *
 * Description:
 *     This classifies block error status into non-retryable errors and ones
 *     that may be successful if retried on a failover path.
 *
 * Return:
 *     %false - retrying failover path will not help
 *     %true  - may succeed if retried
 */
static inline bool blk_path_error(blk_status_t error)
{
        switch (error) {
        case BLK_STS_NOTSUPP:
        case BLK_STS_NOSPC:
        case BLK_STS_TARGET:
        case BLK_STS_RESV_CONFLICT:
        case BLK_STS_MEDIUM:
        case BLK_STS_PROTECTION:
                return false;
        }

        /* Anything else could be a path failure, so should be retried */
        return true;
}

struct bio_issue {
        u64 value;
};

typedef __u32 __bitwise blk_opf_t;

typedef unsigned int blk_qc_t;
#define BLK_QC_T_NONE                -1U

/*
 * main unit of I/O for the block layer and lower layers (ie drivers and
 * stacking drivers)
 */
struct bio {
        struct bio                *bi_next;        /* request queue link */
        struct block_device        *bi_bdev;
        blk_opf_t                bi_opf;                /* bottom bits REQ_OP, top bits
                                                 * req_flags.
                                                 */
        unsigned short                bi_flags;        /* BIO_* below */
        unsigned short                bi_ioprio;
        enum rw_hint                bi_write_hint;
        blk_status_t                bi_status;
        atomic_t                __bi_remaining;

        struct bvec_iter        bi_iter;

        union {
                /* for polled bios: */
                blk_qc_t                bi_cookie;
                /* for plugged zoned writes only: */
                unsigned int                __bi_nr_segments;
        };
        bio_end_io_t                *bi_end_io;
        void                        *bi_private;
#ifdef CONFIG_BLK_CGROUP
        /*
         * Represents the association of the css and request_queue for the bio.
         * If a bio goes direct to device, it will not have a blkg as it will
         * not have a request_queue associated with it.  The reference is put
         * on release of the bio.
         */
        struct blkcg_gq                *bi_blkg;
        struct bio_issue        bi_issue;
#ifdef CONFIG_BLK_CGROUP_IOCOST
        u64                        bi_iocost_cost;
#endif
#endif

#ifdef CONFIG_BLK_INLINE_ENCRYPTION
        struct bio_crypt_ctx        *bi_crypt_context;
#endif

#if defined(CONFIG_BLK_DEV_INTEGRITY)
        struct bio_integrity_payload *bi_integrity; /* data integrity */
#endif

        unsigned short                bi_vcnt;        /* how many bio_vec's */

        /*
         * Everything starting with bi_max_vecs will be preserved by bio_reset()
         */

        unsigned short                bi_max_vecs;        /* max bvl_vecs we can hold */

        atomic_t                __bi_cnt;        /* pin count */

        struct bio_vec                *bi_io_vec;        /* the actual vec list */

        struct bio_set                *bi_pool;

        /*
         * We can inline a number of vecs at the end of the bio, to avoid
         * double allocations for a small number of bio_vecs. This member
         * MUST obviously be kept at the very end of the bio.
         */
        struct bio_vec                bi_inline_vecs[];
};

#define BIO_RESET_BYTES                offsetof(struct bio, bi_max_vecs)
#define BIO_MAX_SECTORS                (UINT_MAX >> SECTOR_SHIFT)

/*
 * bio flags
 */
enum {
        BIO_PAGE_PINNED,        /* Unpin pages in bio_release_pages() */
        BIO_CLONED,                /* doesn't own data */
        BIO_BOUNCED,                /* bio is a bounce bio */
        BIO_QUIET,                /* Make BIO Quiet */
        BIO_CHAIN,                /* chained bio, ->bi_remaining in effect */
        BIO_REFFED,                /* bio has elevated ->bi_cnt */
        BIO_BPS_THROTTLED,        /* This bio has already been subjected to
                                 * throttling rules. Don't do it again. */
        BIO_TRACE_COMPLETION,        /* bio_endio() should trace the final completion
                                 * of this bio. */
        BIO_CGROUP_ACCT,        /* has been accounted to a cgroup */
        BIO_QOS_THROTTLED,        /* bio went through rq_qos throttle path */
        BIO_QOS_MERGED,                /* but went through rq_qos merge path */
        BIO_REMAPPED,
        BIO_ZONE_WRITE_PLUGGING, /* bio handled through zone write plugging */
        BIO_EMULATES_ZONE_APPEND, /* bio emulates a zone append operation */
        BIO_FLAG_LAST
};

typedef __u32 __bitwise blk_mq_req_flags_t;

#define REQ_OP_BITS        8
#define REQ_OP_MASK        (__force blk_opf_t)((1 << REQ_OP_BITS) - 1)
#define REQ_FLAG_BITS        24

/**
 * enum req_op - Operations common to the bio and request structures.
 * We use 8 bits for encoding the operation, and the remaining 24 for flags.
 *
 * The least significant bit of the operation number indicates the data
 * transfer direction:
 *
 *   - if the least significant bit is set transfers are TO the device
 *   - if the least significant bit is not set transfers are FROM the device
 *
 * If a operation does not transfer data the least significant bit has no
 * meaning.
 */
enum req_op {
        /* read sectors from the device */
        REQ_OP_READ                = (__force blk_opf_t)0,
        /* write sectors to the device */
        REQ_OP_WRITE                = (__force blk_opf_t)1,
        /* flush the volatile write cache */
        REQ_OP_FLUSH                = (__force blk_opf_t)2,
        /* discard sectors */
        REQ_OP_DISCARD                = (__force blk_opf_t)3,
        /* securely erase sectors */
        REQ_OP_SECURE_ERASE        = (__force blk_opf_t)5,
        /* write data at the current zone write pointer */
        REQ_OP_ZONE_APPEND        = (__force blk_opf_t)7,
        /* write the zero filled sector many times */
        REQ_OP_WRITE_ZEROES        = (__force blk_opf_t)9,
        /* Open a zone */
        REQ_OP_ZONE_OPEN        = (__force blk_opf_t)10,
        /* Close a zone */
        REQ_OP_ZONE_CLOSE        = (__force blk_opf_t)11,
        /* Transition a zone to full */
        REQ_OP_ZONE_FINISH        = (__force blk_opf_t)12,
        /* reset a zone write pointer */
        REQ_OP_ZONE_RESET        = (__force blk_opf_t)13,
        /* reset all the zone present on the device */
        REQ_OP_ZONE_RESET_ALL        = (__force blk_opf_t)15,

        /* Driver private requests */
        REQ_OP_DRV_IN                = (__force blk_opf_t)34,
        REQ_OP_DRV_OUT                = (__force blk_opf_t)35,

        REQ_OP_LAST                = (__force blk_opf_t)36,
};

/* Keep cmd_flag_name[] in sync with the definitions below */
enum req_flag_bits {
        __REQ_FAILFAST_DEV =        /* no driver retries of device errors */
                REQ_OP_BITS,
        __REQ_FAILFAST_TRANSPORT, /* no driver retries of transport errors */
        __REQ_FAILFAST_DRIVER,        /* no driver retries of driver errors */
        __REQ_SYNC,                /* request is sync (sync write or read) */
        __REQ_META,                /* metadata io request */
        __REQ_PRIO,                /* boost priority in cfq */
        __REQ_NOMERGE,                /* don't touch this for merging */
        __REQ_IDLE,                /* anticipate more IO after this one */
        __REQ_INTEGRITY,        /* I/O includes block integrity payload */
        __REQ_FUA,                /* forced unit access */
        __REQ_PREFLUSH,                /* request for cache flush */
        __REQ_RAHEAD,                /* read ahead, can fail anytime */
        __REQ_BACKGROUND,        /* background IO */
        __REQ_NOWAIT,           /* Don't wait if request will block */
        __REQ_POLLED,                /* caller polls for completion using bio_poll */
        __REQ_ALLOC_CACHE,        /* allocate IO from cache if available */
        __REQ_SWAP,                /* swap I/O */
        __REQ_DRV,                /* for driver use */
        __REQ_FS_PRIVATE,        /* for file system (submitter) use */
        __REQ_ATOMIC,                /* for atomic write operations */
        /*
         * Command specific flags, keep last:
         */
        /* for REQ_OP_WRITE_ZEROES: */
        __REQ_NOUNMAP,                /* do not free blocks when zeroing */

        __REQ_NR_BITS,                /* stops here */
};

#define REQ_FAILFAST_DEV        \
                        (__force blk_opf_t)(1ULL << __REQ_FAILFAST_DEV)
#define REQ_FAILFAST_TRANSPORT        \
                        (__force blk_opf_t)(1ULL << __REQ_FAILFAST_TRANSPORT)
#define REQ_FAILFAST_DRIVER        \
                        (__force blk_opf_t)(1ULL << __REQ_FAILFAST_DRIVER)
#define REQ_SYNC        (__force blk_opf_t)(1ULL << __REQ_SYNC)
#define REQ_META        (__force blk_opf_t)(1ULL << __REQ_META)
#define REQ_PRIO        (__force blk_opf_t)(1ULL << __REQ_PRIO)
#define REQ_NOMERGE        (__force blk_opf_t)(1ULL << __REQ_NOMERGE)
#define REQ_IDLE        (__force blk_opf_t)(1ULL << __REQ_IDLE)
#define REQ_INTEGRITY        (__force blk_opf_t)(1ULL << __REQ_INTEGRITY)
#define REQ_FUA                (__force blk_opf_t)(1ULL << __REQ_FUA)
#define REQ_PREFLUSH        (__force blk_opf_t)(1ULL << __REQ_PREFLUSH)
#define REQ_RAHEAD        (__force blk_opf_t)(1ULL << __REQ_RAHEAD)
#define REQ_BACKGROUND        (__force blk_opf_t)(1ULL << __REQ_BACKGROUND)
#define REQ_NOWAIT        (__force blk_opf_t)(1ULL << __REQ_NOWAIT)
#define REQ_POLLED        (__force blk_opf_t)(1ULL << __REQ_POLLED)
#define REQ_ALLOC_CACHE        (__force blk_opf_t)(1ULL << __REQ_ALLOC_CACHE)
#define REQ_SWAP        (__force blk_opf_t)(1ULL << __REQ_SWAP)
#define REQ_DRV                (__force blk_opf_t)(1ULL << __REQ_DRV)
#define REQ_FS_PRIVATE        (__force blk_opf_t)(1ULL << __REQ_FS_PRIVATE)
#define REQ_ATOMIC        (__force blk_opf_t)(1ULL << __REQ_ATOMIC)

#define REQ_NOUNMAP        (__force blk_opf_t)(1ULL << __REQ_NOUNMAP)

#define REQ_FAILFAST_MASK \
        (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)

#define REQ_NOMERGE_FLAGS \
        (REQ_NOMERGE | REQ_PREFLUSH | REQ_FUA)

enum stat_group {
        STAT_READ,
        STAT_WRITE,
        STAT_DISCARD,
        STAT_FLUSH,

        NR_STAT_GROUPS
};

static inline enum req_op bio_op(const struct bio *bio)
{
        return bio->bi_opf & REQ_OP_MASK;
}

static inline bool op_is_write(blk_opf_t op)
{
        return !!(op & (__force blk_opf_t)1);
}

/*
 * Check if the bio or request is one that needs special treatment in the
 * flush state machine.
 */
static inline bool op_is_flush(blk_opf_t op)
{
        return op & (REQ_FUA | REQ_PREFLUSH);
}

/*
 * Reads are always treated as synchronous, as are requests with the FUA or
 * PREFLUSH flag.  Other operations may be marked as synchronous using the
 * REQ_SYNC flag.
 */
static inline bool op_is_sync(blk_opf_t op)
{
        return (op & REQ_OP_MASK) == REQ_OP_READ ||
                (op & (REQ_SYNC | REQ_FUA | REQ_PREFLUSH));
}

static inline bool op_is_discard(blk_opf_t op)
{
        return (op & REQ_OP_MASK) == REQ_OP_DISCARD;
}

/*
 * Check if a bio or request operation is a zone management operation, with
 * the exception of REQ_OP_ZONE_RESET_ALL which is treated as a special case
 * due to its different handling in the block layer and device response in
 * case of command failure.
 */
static inline bool op_is_zone_mgmt(enum req_op op)
{
        switch (op & REQ_OP_MASK) {
        case REQ_OP_ZONE_RESET:
        case REQ_OP_ZONE_OPEN:
        case REQ_OP_ZONE_CLOSE:
        case REQ_OP_ZONE_FINISH:
                return true;
        default:
                return false;
        }
}

static inline int op_stat_group(enum req_op op)
{
        if (op_is_discard(op))
                return STAT_DISCARD;
        return op_is_write(op);
}

struct blk_rq_stat {
        u64 mean;
        u64 min;
        u64 max;
        u32 nr_samples;
        u64 batch;
};

#endif /* __LINUX_BLK_TYPES_H */


















   70 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PATH_H
#define _LINUX_PATH_H

struct dentry;
struct vfsmount;

struct path {
        struct vfsmount *mnt;
        struct dentry *dentry;
} __randomize_layout;

extern void path_get(const struct path *);
extern void path_put(const struct path *);

static inline int path_equal(const struct path *path1, const struct path *path2)
{
        return path1->mnt == path2->mnt && path1->dentry == path2->dentry;
}

/*
 * Cleanup macro for use with __free(path_put). Avoids dereference and
 * copying @path unlike DEFINE_FREE(). path_put() will handle the empty
 * path correctly just ensure @path is initialized:
 *
 * struct path path __free(path_put) = {};
 */
#define __free_path_put path_put

#endif  /* _LINUX_PATH_H */














































    2 





    2 


    2 

    2 








    2 







    2 















    2 





    2 








    2 





















    2 
    2 





    1 




















   17 



   17 

   17 
































   17 


   17 
   17 





















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
/*
 * Copyright (C) 2015 Red Hat, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include <drm/drm_file.h>
#include <drm/drm_fourcc.h>

#include "virtgpu_drv.h"

static int virtio_gpu_gem_create(struct drm_file *file,
                                 struct drm_device *dev,
                                 struct virtio_gpu_object_params *params,
                                 struct drm_gem_object **obj_p,
                                 uint32_t *handle_p)
{
        struct virtio_gpu_device *vgdev = dev->dev_private;
        struct virtio_gpu_object *obj;
        int ret;
        u32 handle;

        ret = virtio_gpu_object_create(vgdev, params, &obj, NULL);
        if (ret < 0)
                return ret;

        ret = drm_gem_handle_create(file, &obj->base.base, &handle);
        if (ret) {
                drm_gem_object_release(&obj->base.base);
                return ret;
        }

        *obj_p = &obj->base.base;

        /* drop reference from allocate - handle holds it now */
        drm_gem_object_put(&obj->base.base);

        *handle_p = handle;
        return 0;
}

int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
                                struct drm_device *dev,
                                struct drm_mode_create_dumb *args)
{
        struct drm_gem_object *gobj;
        struct virtio_gpu_object_params params = { 0 };
        struct virtio_gpu_device *vgdev = dev->dev_private;
        int ret;
        uint32_t pitch;

        if (args->bpp != 32)
                return -EINVAL;

        pitch = args->width * 4;
        args->size = pitch * args->height;
        args->size = ALIGN(args->size, PAGE_SIZE);

        params.format = virtio_gpu_translate_format(DRM_FORMAT_HOST_XRGB8888);
        params.width = args->width;
        params.height = args->height;
        params.size = args->size;
        params.dumb = true;

        if (vgdev->has_resource_blob && !vgdev->has_virgl_3d) {
                params.blob_mem = VIRTGPU_BLOB_MEM_GUEST;
                params.blob_flags = VIRTGPU_BLOB_FLAG_USE_SHAREABLE;
                params.blob = true;
        }

        ret = virtio_gpu_gem_create(file_priv, dev, &params, &gobj,
                                    &args->handle);
        if (ret)
                goto fail;

        args->pitch = pitch;
        return ret;

fail:
        return ret;
}

int virtio_gpu_gem_object_open(struct drm_gem_object *obj,
                               struct drm_file *file)
{
        struct virtio_gpu_device *vgdev = obj->dev->dev_private;
        struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
        struct virtio_gpu_object_array *objs;

        if (!vgdev->has_virgl_3d)
                goto out_notify;

        /* the context might still be missing when the first ioctl is
         * DRM_IOCTL_MODE_CREATE_DUMB or DRM_IOCTL_PRIME_FD_TO_HANDLE
         */
        if (!vgdev->has_context_init)
                virtio_gpu_create_context(obj->dev, file);

        objs = virtio_gpu_array_alloc(1);
        if (!objs)
                return -ENOMEM;
        virtio_gpu_array_add_obj(objs, obj);

        if (vfpriv->ctx_id)
                virtio_gpu_cmd_context_attach_resource(vgdev, vfpriv->ctx_id, objs);

out_notify:
        virtio_gpu_notify(vgdev);
        return 0;
}

void virtio_gpu_gem_object_close(struct drm_gem_object *obj,
                                 struct drm_file *file)
{
        struct virtio_gpu_device *vgdev = obj->dev->dev_private;
        struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
        struct virtio_gpu_object_array *objs;

        if (!vgdev->has_virgl_3d)
                return;

        objs = virtio_gpu_array_alloc(1);
        if (!objs)
                return;
        virtio_gpu_array_add_obj(objs, obj);

        virtio_gpu_cmd_context_detach_resource(vgdev, vfpriv->ctx_id,
                                               objs);
        virtio_gpu_notify(vgdev);
}

struct virtio_gpu_object_array *virtio_gpu_array_alloc(u32 nents)
{
        struct virtio_gpu_object_array *objs;

        objs = kmalloc(struct_size(objs, objs, nents), GFP_KERNEL);
        if (!objs)
                return NULL;

        objs->nents = 0;
        objs->total = nents;
        return objs;
}

static void virtio_gpu_array_free(struct virtio_gpu_object_array *objs)
{
        kfree(objs);
}

struct virtio_gpu_object_array*
virtio_gpu_array_from_handles(struct drm_file *drm_file, u32 *handles, u32 nents)
{
        struct virtio_gpu_object_array *objs;
        u32 i;

        objs = virtio_gpu_array_alloc(nents);
        if (!objs)
                return NULL;

        for (i = 0; i < nents; i++) {
                objs->objs[i] = drm_gem_object_lookup(drm_file, handles[i]);
                if (!objs->objs[i]) {
                        objs->nents = i;
                        virtio_gpu_array_put_free(objs);
                        return NULL;
                }
        }
        objs->nents = i;
        return objs;
}

void virtio_gpu_array_add_obj(struct virtio_gpu_object_array *objs,
                              struct drm_gem_object *obj)
{
        if (WARN_ON_ONCE(objs->nents == objs->total))
                return;

        drm_gem_object_get(obj);
        objs->objs[objs->nents] = obj;
        objs->nents++;
}

int virtio_gpu_array_lock_resv(struct virtio_gpu_object_array *objs)
{
        unsigned int i;
        int ret;

        if (objs->nents == 1) {
                ret = dma_resv_lock_interruptible(objs->objs[0]->resv, NULL);
        } else {
                ret = drm_gem_lock_reservations(objs->objs, objs->nents,
                                                &objs->ticket);
        }
        if (ret)
                return ret;

        for (i = 0; i < objs->nents; ++i) {
                ret = dma_resv_reserve_fences(objs->objs[i]->resv, 1);
                if (ret) {
                        virtio_gpu_array_unlock_resv(objs);
                        return ret;
                }
        }
        return ret;
}

void virtio_gpu_array_unlock_resv(struct virtio_gpu_object_array *objs)
{
        if (objs->nents == 1) {
                dma_resv_unlock(objs->objs[0]->resv);
        } else {
                drm_gem_unlock_reservations(objs->objs, objs->nents,
                                            &objs->ticket);
        }
}

void virtio_gpu_array_add_fence(struct virtio_gpu_object_array *objs,
                                struct dma_fence *fence)
{
        int i;

        for (i = 0; i < objs->nents; i++)
                dma_resv_add_fence(objs->objs[i]->resv, fence,
                                   DMA_RESV_USAGE_WRITE);
}

void virtio_gpu_array_put_free(struct virtio_gpu_object_array *objs)
{
        u32 i;

        if (!objs)
                return;

        for (i = 0; i < objs->nents; i++)
                drm_gem_object_put(objs->objs[i]);
        virtio_gpu_array_free(objs);
}

void virtio_gpu_array_put_free_delayed(struct virtio_gpu_device *vgdev,
                                       struct virtio_gpu_object_array *objs)
{
        spin_lock(&vgdev->obj_free_lock);
        list_add_tail(&objs->next, &vgdev->obj_free_list);
        spin_unlock(&vgdev->obj_free_lock);
        schedule_work(&vgdev->obj_free_work);
}

void virtio_gpu_array_put_free_work(struct work_struct *work)
{
        struct virtio_gpu_device *vgdev =
                container_of(work, struct virtio_gpu_device, obj_free_work);
        struct virtio_gpu_object_array *objs;

        spin_lock(&vgdev->obj_free_lock);
        while (!list_empty(&vgdev->obj_free_list)) {
                objs = list_first_entry(&vgdev->obj_free_list,
                                        struct virtio_gpu_object_array, next);
                list_del(&objs->next);
                spin_unlock(&vgdev->obj_free_lock);
                virtio_gpu_array_put_free(objs);
                spin_lock(&vgdev->obj_free_lock);
        }
        spin_unlock(&vgdev->obj_free_lock);
}










































































































































    1 
    1 

    1 





    3 
































    1 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (c) International Business Machines Corp., 2006
 *
 * Author: Artem Bityutskiy (Битюцкий Артём)
 */

/* Here we keep miscellaneous functions which are used all over the UBI code */

#include "ubi.h"

/**
 * ubi_calc_data_len - calculate how much real data is stored in a buffer.
 * @ubi: UBI device description object
 * @buf: a buffer with the contents of the physical eraseblock
 * @length: the buffer length
 *
 * This function calculates how much "real data" is stored in @buf and returnes
 * the length. Continuous 0xFF bytes at the end of the buffer are not
 * considered as "real data".
 */
int ubi_calc_data_len(const struct ubi_device *ubi, const void *buf,
                      int length)
{
        int i;

        ubi_assert(!(length & (ubi->min_io_size - 1)));

        for (i = length - 1; i >= 0; i--)
                if (((const uint8_t *)buf)[i] != 0xFF)
                        break;

        /* The resulting length must be aligned to the minimum flash I/O size */
        length = ALIGN(i + 1, ubi->min_io_size);
        return length;
}

/**
 * ubi_check_volume - check the contents of a static volume.
 * @ubi: UBI device description object
 * @vol_id: ID of the volume to check
 *
 * This function checks if static volume @vol_id is corrupted by fully reading
 * it and checking data CRC. This function returns %0 if the volume is not
 * corrupted, %1 if it is corrupted and a negative error code in case of
 * failure. Dynamic volumes are not checked and zero is returned immediately.
 */
int ubi_check_volume(struct ubi_device *ubi, int vol_id)
{
        void *buf;
        int err = 0, i;
        struct ubi_volume *vol = ubi->volumes[vol_id];

        if (vol->vol_type != UBI_STATIC_VOLUME)
                return 0;

        buf = vmalloc(vol->usable_leb_size);
        if (!buf)
                return -ENOMEM;

        for (i = 0; i < vol->used_ebs; i++) {
                int size;

                cond_resched();

                if (i == vol->used_ebs - 1)
                        size = vol->last_eb_bytes;
                else
                        size = vol->usable_leb_size;

                err = ubi_eba_read_leb(ubi, vol, i, buf, 0, size, 1);
                if (err) {
                        if (mtd_is_eccerr(err))
                                err = 1;
                        break;
                }
        }

        vfree(buf);
        return err;
}

/**
 * ubi_update_reserved - update bad eraseblock handling accounting data.
 * @ubi: UBI device description object
 *
 * This function calculates the gap between current number of PEBs reserved for
 * bad eraseblock handling and the required level of PEBs that must be
 * reserved, and if necessary, reserves more PEBs to fill that gap, according
 * to availability. Should be called with ubi->volumes_lock held.
 */
void ubi_update_reserved(struct ubi_device *ubi)
{
        int need = ubi->beb_rsvd_level - ubi->beb_rsvd_pebs;

        if (need <= 0 || ubi->avail_pebs == 0)
                return;

        need = min_t(int, need, ubi->avail_pebs);
        ubi->avail_pebs -= need;
        ubi->rsvd_pebs += need;
        ubi->beb_rsvd_pebs += need;
        ubi_msg(ubi, "reserved more %d PEBs for bad PEB handling", need);
}

/**
 * ubi_calculate_reserved - calculate how many PEBs must be reserved for bad
 * eraseblock handling.
 * @ubi: UBI device description object
 */
void ubi_calculate_reserved(struct ubi_device *ubi)
{
        /*
         * Calculate the actual number of PEBs currently needed to be reserved
         * for future bad eraseblock handling.
         */
        ubi->beb_rsvd_level = ubi->bad_peb_limit - ubi->bad_peb_count;
        if (ubi->beb_rsvd_level < 0) {
                ubi->beb_rsvd_level = 0;
                ubi_warn(ubi, "number of bad PEBs (%d) is above the expected limit (%d), not reserving any PEBs for bad PEB handling, will use available PEBs (if any)",
                         ubi->bad_peb_count, ubi->bad_peb_limit);
        }
}

/**
 * ubi_check_pattern - check if buffer contains only a certain byte pattern.
 * @buf: buffer to check
 * @patt: the pattern to check
 * @size: buffer size in bytes
 *
 * This function returns %1 in there are only @patt bytes in @buf, and %0 if
 * something else was also found.
 */
int ubi_check_pattern(const void *buf, uint8_t patt, int size)
{
        int i;

        for (i = 0; i < size; i++)
                if (((const uint8_t *)buf)[i] != patt)
                        return 0;
        return 1;
}

/* Normal UBI messages */
void ubi_msg(const struct ubi_device *ubi, const char *fmt, ...)
{
        struct va_format vaf;
        va_list args;

        va_start(args, fmt);

        vaf.fmt = fmt;
        vaf.va = &args;

        pr_notice(UBI_NAME_STR "%d: %pV\n", ubi->ubi_num, &vaf);

        va_end(args);
}

/* UBI warning messages */
void ubi_warn(const struct ubi_device *ubi, const char *fmt, ...)
{
        struct va_format vaf;
        va_list args;

        va_start(args, fmt);

        vaf.fmt = fmt;
        vaf.va = &args;

        pr_warn(UBI_NAME_STR "%d warning: %ps: %pV\n",
                ubi->ubi_num, __builtin_return_address(0), &vaf);

        va_end(args);
}

/* UBI error messages */
void ubi_err(const struct ubi_device *ubi, const char *fmt, ...)
{
        struct va_format vaf;
        va_list args;

        va_start(args, fmt);

        vaf.fmt = fmt;
        vaf.va = &args;

        pr_err(UBI_NAME_STR "%d error: %ps: %pV\n",
               ubi->ubi_num, __builtin_return_address(0), &vaf);
        va_end(args);
}

























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    1 
















    1 


    1 


    3 












    1 


    1 


    3 












    3 




    1 


    1 


    3 













    2 





    2 


    2 














    1 


    1 














    1 






    2 






    6 


    6 
    2 




    6 

    4 















    3 










    3 
    3 





    1 
    3 

    3 




    3 
    2 







    3 


    4 
    2 

















    3 










    3 






    3 









    3 
    1 



    2 

    3 
    1 


































































































































































































   11 




   11 
   11 








   11 
   11 


    4 

   10 
    9 


   10 
   10 
   11 
    9 

   11 

    1 



   10 



    3 


   10 
    3 



    5 



    3 


    3 


    3 

    3 


    3 











    3 



    3 
    3 








    6 




    6 
    6 



    3 




    6 

    3 

    5 



    4 









    4 
    2 

    2 
    4 












    3 


    3 
    3 



    3 
    3 






    3 
    3 
    3 
    3 

    3 



    3 



    5 




   11 











    4 


    2 

    1 














    1 
    1 










    1 


    2 

    2 



    1 

    1 


    1 






    1 



    1 
    2 



    2 
    2 
    2 

    2 






    3 
    4 







    9 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
// SPDX-License-Identifier: GPL-2.0
/*
 * IPVS         An implementation of the IP virtual server support for the
 *              LINUX operating system.  IPVS is now implemented as a module
 *              over the NetFilter framework. IPVS can be used to build a
 *              high-performance and highly available server based on a
 *              cluster of servers.
 *
 * Version 1,   is capable of handling both version 0 and 1 messages.
 *              Version 0 is the plain old format.
 *              Note Version 0 receivers will just drop Ver 1 messages.
 *              Version 1 is capable of handle IPv6, Persistence data,
 *              time-outs, and firewall marks.
 *              In ver.1 "ip_vs_sync_conn_options" will be sent in netw. order.
 *              Ver. 0 can be turned on by sysctl -w net.ipv4.vs.sync_version=0
 *
 * Definitions  Message: is a complete datagram
 *              Sync_conn: is a part of a Message
 *              Param Data is an option to a Sync_conn.
 *
 * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
 *
 * ip_vs_sync:  sync connection info from master load balancer to backups
 *              through multicast
 *
 * Changes:
 *        Alexandre Cassen        :        Added master & backup support at a time.
 *        Alexandre Cassen        :        Added SyncID support for incoming sync
 *                                        messages filtering.
 *        Justin Ossevoort        :        Fix endian problem on sync message size.
 *        Hans Schillstrom        :        Added Version 1: i.e. IPv6,
 *                                        Persistence support, fwmark and time-out.
 */

#define KMSG_COMPONENT "IPVS"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/inetdevice.h>
#include <linux/net.h>
#include <linux/completion.h>
#include <linux/delay.h>
#include <linux/skbuff.h>
#include <linux/in.h>
#include <linux/igmp.h>                 /* for ip_mc_join_group */
#include <linux/udp.h>
#include <linux/err.h>
#include <linux/kthread.h>
#include <linux/wait.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>

#include <linux/unaligned.h>                /* Used for ntoh_seq and hton_seq */

#include <net/ip.h>
#include <net/sock.h>

#include <net/ip_vs.h>

#define IP_VS_SYNC_GROUP 0xe0000051    /* multicast addr - 224.0.0.81 */
#define IP_VS_SYNC_PORT  8848          /* multicast port */

#define SYNC_PROTO_VER  1                /* Protocol version in header */

static struct lock_class_key __ipvs_sync_key;
/*
 *        IPVS sync connection entry
 *        Version 0, i.e. original version.
 */
struct ip_vs_sync_conn_v0 {
        __u8                        reserved;

        /* Protocol, addresses and port numbers */
        __u8                        protocol;       /* Which protocol (TCP/UDP) */
        __be16                        cport;
        __be16                  vport;
        __be16                  dport;
        __be32                  caddr;          /* client address */
        __be32                  vaddr;          /* virtual address */
        __be32                  daddr;          /* destination address */

        /* Flags and state transition */
        __be16                  flags;          /* status flags */
        __be16                  state;          /* state info */

        /* The sequence options start here */
};

struct ip_vs_sync_conn_options {
        struct ip_vs_seq        in_seq;         /* incoming seq. struct */
        struct ip_vs_seq        out_seq;        /* outgoing seq. struct */
};

/*
     Sync Connection format (sync_conn)

       0                   1                   2                   3
       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |    Type       |    Protocol   | Ver.  |        Size           |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                             Flags                             |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |            State              |         cport                 |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |            vport              |         dport                 |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                             fwmark                            |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                             timeout  (in sec.)                |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                              ...                              |
      |                        IP-Addresses  (v4 or v6)               |
      |                              ...                              |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  Optional Parameters.
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      | Param. Type    | Param. Length |   Param. data                |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+                               |
      |                              ...                              |
      |                               +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                               | Param Type    | Param. Length |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                           Param  data                         |
      |         Last Param data should be padded for 32 bit alignment |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/

/*
 *  Type 0, IPv4 sync connection format
 */
struct ip_vs_sync_v4 {
        __u8                        type;
        __u8                        protocol;        /* Which protocol (TCP/UDP) */
        __be16                        ver_size;        /* Version msb 4 bits */
        /* Flags and state transition */
        __be32                        flags;                /* status flags */
        __be16                        state;                /* state info         */
        /* Protocol, addresses and port numbers */
        __be16                        cport;
        __be16                        vport;
        __be16                        dport;
        __be32                        fwmark;                /* Firewall mark from skb */
        __be32                        timeout;        /* cp timeout */
        __be32                        caddr;                /* client address */
        __be32                        vaddr;                /* virtual address */
        __be32                        daddr;                /* destination address */
        /* The sequence options start here */
        /* PE data padded to 32bit alignment after seq. options */
};
/*
 * Type 2 messages IPv6
 */
struct ip_vs_sync_v6 {
        __u8                        type;
        __u8                        protocol;        /* Which protocol (TCP/UDP) */
        __be16                        ver_size;        /* Version msb 4 bits */
        /* Flags and state transition */
        __be32                        flags;                /* status flags */
        __be16                        state;                /* state info         */
        /* Protocol, addresses and port numbers */
        __be16                        cport;
        __be16                        vport;
        __be16                        dport;
        __be32                        fwmark;                /* Firewall mark from skb */
        __be32                        timeout;        /* cp timeout */
        struct in6_addr                caddr;                /* client address */
        struct in6_addr                vaddr;                /* virtual address */
        struct in6_addr                daddr;                /* destination address */
        /* The sequence options start here */
        /* PE data padded to 32bit alignment after seq. options */
};

union ip_vs_sync_conn {
        struct ip_vs_sync_v4        v4;
        struct ip_vs_sync_v6        v6;
};

/* Bits in Type field in above */
#define STYPE_INET6                0
#define STYPE_F_INET6                (1 << STYPE_INET6)

#define SVER_SHIFT                12                /* Shift to get version */
#define SVER_MASK                0x0fff                /* Mask to strip version */

#define IPVS_OPT_SEQ_DATA        1
#define IPVS_OPT_PE_DATA        2
#define IPVS_OPT_PE_NAME        3
#define IPVS_OPT_PARAM                7

#define IPVS_OPT_F_SEQ_DATA        (1 << (IPVS_OPT_SEQ_DATA-1))
#define IPVS_OPT_F_PE_DATA        (1 << (IPVS_OPT_PE_DATA-1))
#define IPVS_OPT_F_PE_NAME        (1 << (IPVS_OPT_PE_NAME-1))
#define IPVS_OPT_F_PARAM        (1 << (IPVS_OPT_PARAM-1))

struct ip_vs_sync_thread_data {
        struct task_struct *task;
        struct netns_ipvs *ipvs;
        struct socket *sock;
        char *buf;
        int id;
};

/* Version 0 definition of packet sizes */
#define SIMPLE_CONN_SIZE  (sizeof(struct ip_vs_sync_conn_v0))
#define FULL_CONN_SIZE  \
(sizeof(struct ip_vs_sync_conn_v0) + sizeof(struct ip_vs_sync_conn_options))


/*
  The master mulitcasts messages (Datagrams) to the backup load balancers
  in the following format.

 Version 1:
  Note, first byte should be Zero, so ver 0 receivers will drop the packet.

       0                   1                   2                   3
       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |      0        |    SyncID     |            Size               |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |  Count Conns  |    Version    |    Reserved, set to Zero      |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                                                               |
      |                    IPVS Sync Connection (1)                   |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                            .                                  |
      ~                            .                                  ~
      |                            .                                  |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                                                               |
      |                    IPVS Sync Connection (n)                   |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

 Version 0 Header
       0                   1                   2                   3
       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |  Count Conns  |    SyncID     |            Size               |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                    IPVS Sync Connection (1)                   |
*/

/* Version 0 header */
struct ip_vs_sync_mesg_v0 {
        __u8                    nr_conns;
        __u8                    syncid;
        __be16                  size;

        /* ip_vs_sync_conn entries start here */
};

/* Version 1 header */
struct ip_vs_sync_mesg {
        __u8                        reserved;        /* must be zero */
        __u8                        syncid;
        __be16                        size;
        __u8                        nr_conns;
        __s8                        version;        /* SYNC_PROTO_VER  */
        __u16                        spare;
        /* ip_vs_sync_conn entries start here */
};

union ipvs_sockaddr {
        struct sockaddr_in        in;
        struct sockaddr_in6        in6;
};

struct ip_vs_sync_buff {
        struct list_head        list;
        unsigned long           firstuse;

        /* pointers for the message data */
        struct ip_vs_sync_mesg  *mesg;
        unsigned char           *head;
        unsigned char           *end;
};

/*
 * Copy of struct ip_vs_seq
 * From unaligned network order to aligned host order
 */
static void ntoh_seq(struct ip_vs_seq *no, struct ip_vs_seq *ho)
{
        memset(ho, 0, sizeof(*ho));
        ho->init_seq       = get_unaligned_be32(&no->init_seq);
        ho->delta          = get_unaligned_be32(&no->delta);
        ho->previous_delta = get_unaligned_be32(&no->previous_delta);
}

/*
 * Copy of struct ip_vs_seq
 * From Aligned host order to unaligned network order
 */
static void hton_seq(struct ip_vs_seq *ho, struct ip_vs_seq *no)
{
        put_unaligned_be32(ho->init_seq, &no->init_seq);
        put_unaligned_be32(ho->delta, &no->delta);
        put_unaligned_be32(ho->previous_delta, &no->previous_delta);
}

static inline struct ip_vs_sync_buff *
sb_dequeue(struct netns_ipvs *ipvs, struct ipvs_master_sync_state *ms)
{
        struct ip_vs_sync_buff *sb;

        spin_lock_bh(&ipvs->sync_lock);
        if (list_empty(&ms->sync_queue)) {
                sb = NULL;
                __set_current_state(TASK_INTERRUPTIBLE);
        } else {
                sb = list_entry(ms->sync_queue.next, struct ip_vs_sync_buff,
                                list);
                list_del(&sb->list);
                ms->sync_queue_len--;
                if (!ms->sync_queue_len)
                        ms->sync_queue_delay = 0;
        }
        spin_unlock_bh(&ipvs->sync_lock);

        return sb;
}

/*
 * Create a new sync buffer for Version 1 proto.
 */
static inline struct ip_vs_sync_buff *
ip_vs_sync_buff_create(struct netns_ipvs *ipvs, unsigned int len)
{
        struct ip_vs_sync_buff *sb;

        if (!(sb=kmalloc(sizeof(struct ip_vs_sync_buff), GFP_ATOMIC)))
                return NULL;

        len = max_t(unsigned int, len + sizeof(struct ip_vs_sync_mesg),
                    ipvs->mcfg.sync_maxlen);
        sb->mesg = kmalloc(len, GFP_ATOMIC);
        if (!sb->mesg) {
                kfree(sb);
                return NULL;
        }
        sb->mesg->reserved = 0;  /* old nr_conns i.e. must be zero now */
        sb->mesg->version = SYNC_PROTO_VER;
        sb->mesg->syncid = ipvs->mcfg.syncid;
        sb->mesg->size = htons(sizeof(struct ip_vs_sync_mesg));
        sb->mesg->nr_conns = 0;
        sb->mesg->spare = 0;
        sb->head = (unsigned char *)sb->mesg + sizeof(struct ip_vs_sync_mesg);
        sb->end = (unsigned char *)sb->mesg + len;

        sb->firstuse = jiffies;
        return sb;
}

static inline void ip_vs_sync_buff_release(struct ip_vs_sync_buff *sb)
{
        kfree(sb->mesg);
        kfree(sb);
}

static inline void sb_queue_tail(struct netns_ipvs *ipvs,
                                 struct ipvs_master_sync_state *ms)
{
        struct ip_vs_sync_buff *sb = ms->sync_buff;

        spin_lock(&ipvs->sync_lock);
        if (ipvs->sync_state & IP_VS_STATE_MASTER &&
            ms->sync_queue_len < sysctl_sync_qlen_max(ipvs)) {
                if (!ms->sync_queue_len)
                        schedule_delayed_work(&ms->master_wakeup_work,
                                              max(IPVS_SYNC_SEND_DELAY, 1));
                ms->sync_queue_len++;
                list_add_tail(&sb->list, &ms->sync_queue);
                if ((++ms->sync_queue_delay) == IPVS_SYNC_WAKEUP_RATE) {
                        int id = (int)(ms - ipvs->ms);

                        wake_up_process(ipvs->master_tinfo[id].task);
                }
        } else
                ip_vs_sync_buff_release(sb);
        spin_unlock(&ipvs->sync_lock);
}

/*
 *        Get the current sync buffer if it has been created for more
 *        than the specified time or the specified time is zero.
 */
static inline struct ip_vs_sync_buff *
get_curr_sync_buff(struct netns_ipvs *ipvs, struct ipvs_master_sync_state *ms,
                   unsigned long time)
{
        struct ip_vs_sync_buff *sb;

        spin_lock_bh(&ipvs->sync_buff_lock);
        sb = ms->sync_buff;
        if (sb && time_after_eq(jiffies - sb->firstuse, time)) {
                ms->sync_buff = NULL;
                __set_current_state(TASK_RUNNING);
        } else
                sb = NULL;
        spin_unlock_bh(&ipvs->sync_buff_lock);
        return sb;
}

static inline int
select_master_thread_id(struct netns_ipvs *ipvs, struct ip_vs_conn *cp)
{
        return ((long) cp >> (1 + ilog2(sizeof(*cp)))) & ipvs->threads_mask;
}

/*
 * Create a new sync buffer for Version 0 proto.
 */
static inline struct ip_vs_sync_buff *
ip_vs_sync_buff_create_v0(struct netns_ipvs *ipvs, unsigned int len)
{
        struct ip_vs_sync_buff *sb;
        struct ip_vs_sync_mesg_v0 *mesg;

        if (!(sb=kmalloc(sizeof(struct ip_vs_sync_buff), GFP_ATOMIC)))
                return NULL;

        len = max_t(unsigned int, len + sizeof(struct ip_vs_sync_mesg_v0),
                    ipvs->mcfg.sync_maxlen);
        sb->mesg = kmalloc(len, GFP_ATOMIC);
        if (!sb->mesg) {
                kfree(sb);
                return NULL;
        }
        mesg = (struct ip_vs_sync_mesg_v0 *)sb->mesg;
        mesg->nr_conns = 0;
        mesg->syncid = ipvs->mcfg.syncid;
        mesg->size = htons(sizeof(struct ip_vs_sync_mesg_v0));
        sb->head = (unsigned char *)mesg + sizeof(struct ip_vs_sync_mesg_v0);
        sb->end = (unsigned char *)mesg + len;
        sb->firstuse = jiffies;
        return sb;
}

/* Check if connection is controlled by persistence */
static inline bool in_persistence(struct ip_vs_conn *cp)
{
        for (cp = cp->control; cp; cp = cp->control) {
                if (cp->flags & IP_VS_CONN_F_TEMPLATE)
                        return true;
        }
        return false;
}

/* Check if conn should be synced.
 * pkts: conn packets, use sysctl_sync_threshold to avoid packet check
 * - (1) sync_refresh_period: reduce sync rate. Additionally, retry
 *        sync_retries times with period of sync_refresh_period/8
 * - (2) if both sync_refresh_period and sync_period are 0 send sync only
 *        for state changes or only once when pkts matches sync_threshold
 * - (3) templates: rate can be reduced only with sync_refresh_period or
 *        with (2)
 */
static int ip_vs_sync_conn_needed(struct netns_ipvs *ipvs,
                                  struct ip_vs_conn *cp, int pkts)
{
        unsigned long orig = READ_ONCE(cp->sync_endtime);
        unsigned long now = jiffies;
        unsigned long n = (now + cp->timeout) & ~3UL;
        unsigned int sync_refresh_period;
        int sync_period;
        int force;

        /* Check if we sync in current state */
        if (unlikely(cp->flags & IP_VS_CONN_F_TEMPLATE))
                force = 0;
        else if (unlikely(sysctl_sync_persist_mode(ipvs) && in_persistence(cp)))
                return 0;
        else if (likely(cp->protocol == IPPROTO_TCP)) {
                if (!((1 << cp->state) &
                      ((1 << IP_VS_TCP_S_ESTABLISHED) |
                       (1 << IP_VS_TCP_S_FIN_WAIT) |
                       (1 << IP_VS_TCP_S_CLOSE) |
                       (1 << IP_VS_TCP_S_CLOSE_WAIT) |
                       (1 << IP_VS_TCP_S_TIME_WAIT))))
                        return 0;
                force = cp->state != cp->old_state;
                if (force && cp->state != IP_VS_TCP_S_ESTABLISHED)
                        goto set;
        } else if (unlikely(cp->protocol == IPPROTO_SCTP)) {
                if (!((1 << cp->state) &
                      ((1 << IP_VS_SCTP_S_ESTABLISHED) |
                       (1 << IP_VS_SCTP_S_SHUTDOWN_SENT) |
                       (1 << IP_VS_SCTP_S_SHUTDOWN_RECEIVED) |
                       (1 << IP_VS_SCTP_S_SHUTDOWN_ACK_SENT) |
                       (1 << IP_VS_SCTP_S_CLOSED))))
                        return 0;
                force = cp->state != cp->old_state;
                if (force && cp->state != IP_VS_SCTP_S_ESTABLISHED)
                        goto set;
        } else {
                /* UDP or another protocol with single state */
                force = 0;
        }

        sync_refresh_period = sysctl_sync_refresh_period(ipvs);
        if (sync_refresh_period > 0) {
                long diff = n - orig;
                long min_diff = max(cp->timeout >> 1, 10UL * HZ);

                /* Avoid sync if difference is below sync_refresh_period
                 * and below the half timeout.
                 */
                if (abs(diff) < min_t(long, sync_refresh_period, min_diff)) {
                        int retries = orig & 3;

                        if (retries >= sysctl_sync_retries(ipvs))
                                return 0;
                        if (time_before(now, orig - cp->timeout +
                                        (sync_refresh_period >> 3)))
                                return 0;
                        n |= retries + 1;
                }
        }
        sync_period = sysctl_sync_period(ipvs);
        if (sync_period > 0) {
                if (!(cp->flags & IP_VS_CONN_F_TEMPLATE) &&
                    pkts % sync_period != sysctl_sync_threshold(ipvs))
                        return 0;
        } else if (!sync_refresh_period &&
                   pkts != sysctl_sync_threshold(ipvs))
                return 0;

set:
        cp->old_state = cp->state;
        n = cmpxchg(&cp->sync_endtime, orig, n);
        return n == orig || force;
}

/*
 *      Version 0 , could be switched in by sys_ctl.
 *      Add an ip_vs_conn information into the current sync_buff.
 */
static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp,
                               int pkts)
{
        struct ip_vs_sync_mesg_v0 *m;
        struct ip_vs_sync_conn_v0 *s;
        struct ip_vs_sync_buff *buff;
        struct ipvs_master_sync_state *ms;
        int id;
        unsigned int len;

        if (unlikely(cp->af != AF_INET))
                return;
        /* Do not sync ONE PACKET */
        if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
                return;

        if (!ip_vs_sync_conn_needed(ipvs, cp, pkts))
                return;

        spin_lock_bh(&ipvs->sync_buff_lock);
        if (!(ipvs->sync_state & IP_VS_STATE_MASTER)) {
                spin_unlock_bh(&ipvs->sync_buff_lock);
                return;
        }

        id = select_master_thread_id(ipvs, cp);
        ms = &ipvs->ms[id];
        buff = ms->sync_buff;
        len = (cp->flags & IP_VS_CONN_F_SEQ_MASK) ? FULL_CONN_SIZE :
                SIMPLE_CONN_SIZE;
        if (buff) {
                m = (struct ip_vs_sync_mesg_v0 *) buff->mesg;
                /* Send buffer if it is for v1 */
                if (buff->head + len > buff->end || !m->nr_conns) {
                        sb_queue_tail(ipvs, ms);
                        ms->sync_buff = NULL;
                        buff = NULL;
                }
        }
        if (!buff) {
                buff = ip_vs_sync_buff_create_v0(ipvs, len);
                if (!buff) {
                        spin_unlock_bh(&ipvs->sync_buff_lock);
                        pr_err("ip_vs_sync_buff_create failed.\n");
                        return;
                }
                ms->sync_buff = buff;
        }

        m = (struct ip_vs_sync_mesg_v0 *) buff->mesg;
        s = (struct ip_vs_sync_conn_v0 *) buff->head;

        /* copy members */
        s->reserved = 0;
        s->protocol = cp->protocol;
        s->cport = cp->cport;
        s->vport = cp->vport;
        s->dport = cp->dport;
        s->caddr = cp->caddr.ip;
        s->vaddr = cp->vaddr.ip;
        s->daddr = cp->daddr.ip;
        s->flags = htons(cp->flags & ~IP_VS_CONN_F_HASHED);
        s->state = htons(cp->state);
        if (cp->flags & IP_VS_CONN_F_SEQ_MASK) {
                struct ip_vs_sync_conn_options *opt =
                        (struct ip_vs_sync_conn_options *)&s[1];
                memcpy(opt, &cp->sync_conn_opt, sizeof(*opt));
        }

        m->nr_conns++;
        m->size = htons(ntohs(m->size) + len);
        buff->head += len;
        spin_unlock_bh(&ipvs->sync_buff_lock);

        /* synchronize its controller if it has */
        cp = cp->control;
        if (cp) {
                if (cp->flags & IP_VS_CONN_F_TEMPLATE)
                        pkts = atomic_inc_return(&cp->in_pkts);
                else
                        pkts = sysctl_sync_threshold(ipvs);
                ip_vs_sync_conn(ipvs, cp, pkts);
        }
}

/*
 *      Add an ip_vs_conn information into the current sync_buff.
 *      Called by ip_vs_in.
 *      Sending Version 1 messages
 */
void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts)
{
        struct ip_vs_sync_mesg *m;
        union ip_vs_sync_conn *s;
        struct ip_vs_sync_buff *buff;
        struct ipvs_master_sync_state *ms;
        int id;
        __u8 *p;
        unsigned int len, pe_name_len, pad;

        /* Handle old version of the protocol */
        if (sysctl_sync_ver(ipvs) == 0) {
                ip_vs_sync_conn_v0(ipvs, cp, pkts);
                return;
        }
        /* Do not sync ONE PACKET */
        if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
                goto control;
sloop:
        if (!ip_vs_sync_conn_needed(ipvs, cp, pkts))
                goto control;

        /* Sanity checks */
        pe_name_len = 0;
        if (cp->pe_data_len) {
                if (!cp->pe_data || !cp->dest) {
                        IP_VS_ERR_RL("SYNC, connection pe_data invalid\n");
                        return;
                }
                pe_name_len = strnlen(cp->pe->name, IP_VS_PENAME_MAXLEN);
        }

        spin_lock_bh(&ipvs->sync_buff_lock);
        if (!(ipvs->sync_state & IP_VS_STATE_MASTER)) {
                spin_unlock_bh(&ipvs->sync_buff_lock);
                return;
        }

        id = select_master_thread_id(ipvs, cp);
        ms = &ipvs->ms[id];

#ifdef CONFIG_IP_VS_IPV6
        if (cp->af == AF_INET6)
                len = sizeof(struct ip_vs_sync_v6);
        else
#endif
                len = sizeof(struct ip_vs_sync_v4);

        if (cp->flags & IP_VS_CONN_F_SEQ_MASK)
                len += sizeof(struct ip_vs_sync_conn_options) + 2;

        if (cp->pe_data_len)
                len += cp->pe_data_len + 2;        /* + Param hdr field */
        if (pe_name_len)
                len += pe_name_len + 2;

        /* check if there is a space for this one  */
        pad = 0;
        buff = ms->sync_buff;
        if (buff) {
                m = buff->mesg;
                pad = (4 - (size_t) buff->head) & 3;
                /* Send buffer if it is for v0 */
                if (buff->head + len + pad > buff->end || m->reserved) {
                        sb_queue_tail(ipvs, ms);
                        ms->sync_buff = NULL;
                        buff = NULL;
                        pad = 0;
                }
        }

        if (!buff) {
                buff = ip_vs_sync_buff_create(ipvs, len);
                if (!buff) {
                        spin_unlock_bh(&ipvs->sync_buff_lock);
                        pr_err("ip_vs_sync_buff_create failed.\n");
                        return;
                }
                ms->sync_buff = buff;
                m = buff->mesg;
        }

        p = buff->head;
        buff->head += pad + len;
        m->size = htons(ntohs(m->size) + pad + len);
        /* Add ev. padding from prev. sync_conn */
        while (pad--)
                *(p++) = 0;

        s = (union ip_vs_sync_conn *)p;

        /* Set message type  & copy members */
        s->v4.type = (cp->af == AF_INET6 ? STYPE_F_INET6 : 0);
        s->v4.ver_size = htons(len & SVER_MASK);        /* Version 0 */
        s->v4.flags = htonl(cp->flags & ~IP_VS_CONN_F_HASHED);
        s->v4.state = htons(cp->state);
        s->v4.protocol = cp->protocol;
        s->v4.cport = cp->cport;
        s->v4.vport = cp->vport;
        s->v4.dport = cp->dport;
        s->v4.fwmark = htonl(cp->fwmark);
        s->v4.timeout = htonl(cp->timeout / HZ);
        m->nr_conns++;

#ifdef CONFIG_IP_VS_IPV6
        if (cp->af == AF_INET6) {
                p += sizeof(struct ip_vs_sync_v6);
                s->v6.caddr = cp->caddr.in6;
                s->v6.vaddr = cp->vaddr.in6;
                s->v6.daddr = cp->daddr.in6;
        } else
#endif
        {
                p += sizeof(struct ip_vs_sync_v4);        /* options ptr */
                s->v4.caddr = cp->caddr.ip;
                s->v4.vaddr = cp->vaddr.ip;
                s->v4.daddr = cp->daddr.ip;
        }
        if (cp->flags & IP_VS_CONN_F_SEQ_MASK) {
                *(p++) = IPVS_OPT_SEQ_DATA;
                *(p++) = sizeof(struct ip_vs_sync_conn_options);
                hton_seq((struct ip_vs_seq *)p, &cp->in_seq);
                p += sizeof(struct ip_vs_seq);
                hton_seq((struct ip_vs_seq *)p, &cp->out_seq);
                p += sizeof(struct ip_vs_seq);
        }
        /* Handle pe data */
        if (cp->pe_data_len && cp->pe_data) {
                *(p++) = IPVS_OPT_PE_DATA;
                *(p++) = cp->pe_data_len;
                memcpy(p, cp->pe_data, cp->pe_data_len);
                p += cp->pe_data_len;
                if (pe_name_len) {
                        /* Add PE_NAME */
                        *(p++) = IPVS_OPT_PE_NAME;
                        *(p++) = pe_name_len;
                        memcpy(p, cp->pe->name, pe_name_len);
                        p += pe_name_len;
                }
        }

        spin_unlock_bh(&ipvs->sync_buff_lock);

control:
        /* synchronize its controller if it has */
        cp = cp->control;
        if (!cp)
                return;
        if (cp->flags & IP_VS_CONN_F_TEMPLATE)
                pkts = atomic_inc_return(&cp->in_pkts);
        else
                pkts = sysctl_sync_threshold(ipvs);
        goto sloop;
}

/*
 *  fill_param used by version 1
 */
static inline int
ip_vs_conn_fill_param_sync(struct netns_ipvs *ipvs, int af, union ip_vs_sync_conn *sc,
                           struct ip_vs_conn_param *p,
                           __u8 *pe_data, unsigned int pe_data_len,
                           __u8 *pe_name, unsigned int pe_name_len)
{
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                ip_vs_conn_fill_param(ipvs, af, sc->v6.protocol,
                                      (const union nf_inet_addr *)&sc->v6.caddr,
                                      sc->v6.cport,
                                      (const union nf_inet_addr *)&sc->v6.vaddr,
                                      sc->v6.vport, p);
        else
#endif
                ip_vs_conn_fill_param(ipvs, af, sc->v4.protocol,
                                      (const union nf_inet_addr *)&sc->v4.caddr,
                                      sc->v4.cport,
                                      (const union nf_inet_addr *)&sc->v4.vaddr,
                                      sc->v4.vport, p);
        /* Handle pe data */
        if (pe_data_len) {
                if (pe_name_len) {
                        char buff[IP_VS_PENAME_MAXLEN+1];

                        memcpy(buff, pe_name, pe_name_len);
                        buff[pe_name_len]=0;
                        p->pe = __ip_vs_pe_getbyname(buff);
                        if (!p->pe) {
                                IP_VS_DBG(3, "BACKUP, no %s engine found/loaded\n",
                                             buff);
                                return 1;
                        }
                } else {
                        IP_VS_ERR_RL("BACKUP, Invalid PE parameters\n");
                        return 1;
                }

                p->pe_data = kmemdup(pe_data, pe_data_len, GFP_ATOMIC);
                if (!p->pe_data) {
                        module_put(p->pe->module);
                        return -ENOMEM;
                }
                p->pe_data_len = pe_data_len;
        }
        return 0;
}

/*
 *  Connection Add / Update.
 *  Common for version 0 and 1 reception of backup sync_conns.
 *  Param: ...
 *         timeout is in sec.
 */
static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *param,
                            unsigned int flags, unsigned int state,
                            unsigned int protocol, unsigned int type,
                            const union nf_inet_addr *daddr, __be16 dport,
                            unsigned long timeout, __u32 fwmark,
                            struct ip_vs_sync_conn_options *opt)
{
        struct ip_vs_dest *dest;
        struct ip_vs_conn *cp;

        if (!(flags & IP_VS_CONN_F_TEMPLATE)) {
                cp = ip_vs_conn_in_get(param);
                if (cp && ((cp->dport != dport) ||
                           !ip_vs_addr_equal(cp->daf, &cp->daddr, daddr))) {
                        if (!(flags & IP_VS_CONN_F_INACTIVE)) {
                                ip_vs_conn_expire_now(cp);
                                __ip_vs_conn_put(cp);
                                cp = NULL;
                        } else {
                                /* This is the expiration message for the
                                 * connection that was already replaced, so we
                                 * just ignore it.
                                 */
                                __ip_vs_conn_put(cp);
                                kfree(param->pe_data);
                                return;
                        }
                }
        } else {
                cp = ip_vs_ct_in_get(param);
        }

        if (cp) {
                /* Free pe_data */
                kfree(param->pe_data);

                dest = cp->dest;
                spin_lock_bh(&cp->lock);
                if ((cp->flags ^ flags) & IP_VS_CONN_F_INACTIVE &&
                    !(flags & IP_VS_CONN_F_TEMPLATE) && dest) {
                        if (flags & IP_VS_CONN_F_INACTIVE) {
                                atomic_dec(&dest->activeconns);
                                atomic_inc(&dest->inactconns);
                        } else {
                                atomic_inc(&dest->activeconns);
                                atomic_dec(&dest->inactconns);
                        }
                }
                flags &= IP_VS_CONN_F_BACKUP_UPD_MASK;
                flags |= cp->flags & ~IP_VS_CONN_F_BACKUP_UPD_MASK;
                cp->flags = flags;
                spin_unlock_bh(&cp->lock);
                if (!dest)
                        ip_vs_try_bind_dest(cp);
        } else {
                /*
                 * Find the appropriate destination for the connection.
                 * If it is not found the connection will remain unbound
                 * but still handled.
                 */
                rcu_read_lock();
                /* This function is only invoked by the synchronization
                 * code. We do not currently support heterogeneous pools
                 * with synchronization, so we can make the assumption that
                 * the svc_af is the same as the dest_af
                 */
                dest = ip_vs_find_dest(ipvs, type, type, daddr, dport,
                                       param->vaddr, param->vport, protocol,
                                       fwmark, flags);

                cp = ip_vs_conn_new(param, type, daddr, dport, flags, dest,
                                    fwmark);
                rcu_read_unlock();
                if (!cp) {
                        kfree(param->pe_data);
                        IP_VS_DBG(2, "BACKUP, add new conn. failed\n");
                        return;
                }
                if (!(flags & IP_VS_CONN_F_TEMPLATE))
                        kfree(param->pe_data);
        }

        if (opt) {
                cp->in_seq = opt->in_seq;
                cp->out_seq = opt->out_seq;
        }
        atomic_set(&cp->in_pkts, sysctl_sync_threshold(ipvs));
        cp->state = state;
        cp->old_state = cp->state;
        /*
         * For Ver 0 messages style
         *  - Not possible to recover the right timeout for templates
         *  - can not find the right fwmark
         *    virtual service. If needed, we can do it for
         *    non-fwmark persistent services.
         * Ver 1 messages style.
         *  - No problem.
         */
        if (timeout) {
                if (timeout > MAX_SCHEDULE_TIMEOUT / HZ)
                        timeout = MAX_SCHEDULE_TIMEOUT / HZ;
                cp->timeout = timeout*HZ;
        } else {
                struct ip_vs_proto_data *pd;

                pd = ip_vs_proto_data_get(ipvs, protocol);
                if (!(flags & IP_VS_CONN_F_TEMPLATE) && pd && pd->timeout_table)
                        cp->timeout = pd->timeout_table[state];
                else
                        cp->timeout = (3*60*HZ);
        }
        ip_vs_conn_put(cp);
}

/*
 *  Process received multicast message for Version 0
 */
static void ip_vs_process_message_v0(struct netns_ipvs *ipvs, const char *buffer,
                                     const size_t buflen)
{
        struct ip_vs_sync_mesg_v0 *m = (struct ip_vs_sync_mesg_v0 *)buffer;
        struct ip_vs_sync_conn_v0 *s;
        struct ip_vs_sync_conn_options *opt;
        struct ip_vs_protocol *pp;
        struct ip_vs_conn_param param;
        char *p;
        int i;

        p = (char *)buffer + sizeof(struct ip_vs_sync_mesg_v0);
        for (i=0; i<m->nr_conns; i++) {
                unsigned int flags, state;

                if (p + SIMPLE_CONN_SIZE > buffer+buflen) {
                        IP_VS_ERR_RL("BACKUP v0, bogus conn\n");
                        return;
                }
                s = (struct ip_vs_sync_conn_v0 *) p;
                flags = ntohs(s->flags) | IP_VS_CONN_F_SYNC;
                flags &= ~IP_VS_CONN_F_HASHED;
                if (flags & IP_VS_CONN_F_SEQ_MASK) {
                        opt = (struct ip_vs_sync_conn_options *)&s[1];
                        p += FULL_CONN_SIZE;
                        if (p > buffer+buflen) {
                                IP_VS_ERR_RL("BACKUP v0, Dropping buffer bogus conn options\n");
                                return;
                        }
                } else {
                        opt = NULL;
                        p += SIMPLE_CONN_SIZE;
                }

                state = ntohs(s->state);
                if (!(flags & IP_VS_CONN_F_TEMPLATE)) {
                        pp = ip_vs_proto_get(s->protocol);
                        if (!pp) {
                                IP_VS_DBG(2, "BACKUP v0, Unsupported protocol %u\n",
                                        s->protocol);
                                continue;
                        }
                        if (state >= pp->num_states) {
                                IP_VS_DBG(2, "BACKUP v0, Invalid %s state %u\n",
                                        pp->name, state);
                                continue;
                        }
                } else {
                        if (state >= IP_VS_CTPL_S_LAST)
                                IP_VS_DBG(7, "BACKUP v0, Invalid tpl state %u\n",
                                          state);
                }

                ip_vs_conn_fill_param(ipvs, AF_INET, s->protocol,
                                      (const union nf_inet_addr *)&s->caddr,
                                      s->cport,
                                      (const union nf_inet_addr *)&s->vaddr,
                                      s->vport, &param);

                /* Send timeout as Zero */
                ip_vs_proc_conn(ipvs, &param, flags, state, s->protocol, AF_INET,
                                (union nf_inet_addr *)&s->daddr, s->dport,
                                0, 0, opt);
        }
}

/*
 * Handle options
 */
static inline int ip_vs_proc_seqopt(__u8 *p, unsigned int plen,
                                    __u32 *opt_flags,
                                    struct ip_vs_sync_conn_options *opt)
{
        struct ip_vs_sync_conn_options *topt;

        topt = (struct ip_vs_sync_conn_options *)p;

        if (plen != sizeof(struct ip_vs_sync_conn_options)) {
                IP_VS_DBG(2, "BACKUP, bogus conn options length\n");
                return -EINVAL;
        }
        if (*opt_flags & IPVS_OPT_F_SEQ_DATA) {
                IP_VS_DBG(2, "BACKUP, conn options found twice\n");
                return -EINVAL;
        }
        ntoh_seq(&topt->in_seq, &opt->in_seq);
        ntoh_seq(&topt->out_seq, &opt->out_seq);
        *opt_flags |= IPVS_OPT_F_SEQ_DATA;
        return 0;
}

static int ip_vs_proc_str(__u8 *p, unsigned int plen, unsigned int *data_len,
                          __u8 **data, unsigned int maxlen,
                          __u32 *opt_flags, __u32 flag)
{
        if (plen > maxlen) {
                IP_VS_DBG(2, "BACKUP, bogus par.data len > %d\n", maxlen);
                return -EINVAL;
        }
        if (*opt_flags & flag) {
                IP_VS_DBG(2, "BACKUP, Par.data found twice 0x%x\n", flag);
                return -EINVAL;
        }
        *data_len = plen;
        *data = p;
        *opt_flags |= flag;
        return 0;
}
/*
 *   Process a Version 1 sync. connection
 */
static inline int ip_vs_proc_sync_conn(struct netns_ipvs *ipvs, __u8 *p, __u8 *msg_end)
{
        struct ip_vs_sync_conn_options opt;
        union  ip_vs_sync_conn *s;
        struct ip_vs_protocol *pp;
        struct ip_vs_conn_param param;
        __u32 flags;
        unsigned int af, state, pe_data_len=0, pe_name_len=0;
        __u8 *pe_data=NULL, *pe_name=NULL;
        __u32 opt_flags=0;
        int retc=0;

        s = (union ip_vs_sync_conn *) p;

        if (s->v6.type & STYPE_F_INET6) {
#ifdef CONFIG_IP_VS_IPV6
                af = AF_INET6;
                p += sizeof(struct ip_vs_sync_v6);
#else
                IP_VS_DBG(3,"BACKUP, IPv6 msg received, and IPVS is not compiled for IPv6\n");
                retc = 10;
                goto out;
#endif
        } else if (!s->v4.type) {
                af = AF_INET;
                p += sizeof(struct ip_vs_sync_v4);
        } else {
                return -10;
        }
        if (p > msg_end)
                return -20;

        /* Process optional params check Type & Len. */
        while (p < msg_end) {
                int ptype;
                int plen;

                if (p+2 > msg_end)
                        return -30;
                ptype = *(p++);
                plen  = *(p++);

                if (!plen || ((p + plen) > msg_end))
                        return -40;
                /* Handle seq option  p = param data */
                switch (ptype & ~IPVS_OPT_F_PARAM) {
                case IPVS_OPT_SEQ_DATA:
                        if (ip_vs_proc_seqopt(p, plen, &opt_flags, &opt))
                                return -50;
                        break;

                case IPVS_OPT_PE_DATA:
                        if (ip_vs_proc_str(p, plen, &pe_data_len, &pe_data,
                                           IP_VS_PEDATA_MAXLEN, &opt_flags,
                                           IPVS_OPT_F_PE_DATA))
                                return -60;
                        break;

                case IPVS_OPT_PE_NAME:
                        if (ip_vs_proc_str(p, plen,&pe_name_len, &pe_name,
                                           IP_VS_PENAME_MAXLEN, &opt_flags,
                                           IPVS_OPT_F_PE_NAME))
                                return -70;
                        break;

                default:
                        /* Param data mandatory ? */
                        if (!(ptype & IPVS_OPT_F_PARAM)) {
                                IP_VS_DBG(3, "BACKUP, Unknown mandatory param %d found\n",
                                          ptype & ~IPVS_OPT_F_PARAM);
                                retc = 20;
                                goto out;
                        }
                }
                p += plen;  /* Next option */
        }

        /* Get flags and Mask off unsupported */
        flags  = ntohl(s->v4.flags) & IP_VS_CONN_F_BACKUP_MASK;
        flags |= IP_VS_CONN_F_SYNC;
        state = ntohs(s->v4.state);

        if (!(flags & IP_VS_CONN_F_TEMPLATE)) {
                pp = ip_vs_proto_get(s->v4.protocol);
                if (!pp) {
                        IP_VS_DBG(3,"BACKUP, Unsupported protocol %u\n",
                                s->v4.protocol);
                        retc = 30;
                        goto out;
                }
                if (state >= pp->num_states) {
                        IP_VS_DBG(3, "BACKUP, Invalid %s state %u\n",
                                pp->name, state);
                        retc = 40;
                        goto out;
                }
        } else {
                if (state >= IP_VS_CTPL_S_LAST)
                        IP_VS_DBG(7, "BACKUP, Invalid tpl state %u\n",
                                  state);
        }
        if (ip_vs_conn_fill_param_sync(ipvs, af, s, &param, pe_data,
                                       pe_data_len, pe_name, pe_name_len)) {
                retc = 50;
                goto out;
        }
        /* If only IPv4, just silent skip IPv6 */
        if (af == AF_INET)
                ip_vs_proc_conn(ipvs, &param, flags, state, s->v4.protocol, af,
                                (union nf_inet_addr *)&s->v4.daddr, s->v4.dport,
                                ntohl(s->v4.timeout), ntohl(s->v4.fwmark),
                                (opt_flags & IPVS_OPT_F_SEQ_DATA ? &opt : NULL)
                                );
#ifdef CONFIG_IP_VS_IPV6
        else
                ip_vs_proc_conn(ipvs, &param, flags, state, s->v6.protocol, af,
                                (union nf_inet_addr *)&s->v6.daddr, s->v6.dport,
                                ntohl(s->v6.timeout), ntohl(s->v6.fwmark),
                                (opt_flags & IPVS_OPT_F_SEQ_DATA ? &opt : NULL)
                                );
#endif
        ip_vs_pe_put(param.pe);
        return 0;
        /* Error exit */
out:
        IP_VS_DBG(2, "BACKUP, Single msg dropped err:%d\n", retc);
        return retc;

}
/*
 *      Process received multicast message and create the corresponding
 *      ip_vs_conn entries.
 *      Handles Version 0 & 1
 */
static void ip_vs_process_message(struct netns_ipvs *ipvs, __u8 *buffer,
                                  const size_t buflen)
{
        struct ip_vs_sync_mesg *m2 = (struct ip_vs_sync_mesg *)buffer;
        __u8 *p, *msg_end;
        int i, nr_conns;

        if (buflen < sizeof(struct ip_vs_sync_mesg_v0)) {
                IP_VS_DBG(2, "BACKUP, message header too short\n");
                return;
        }

        if (buflen != ntohs(m2->size)) {
                IP_VS_DBG(2, "BACKUP, bogus message size\n");
                return;
        }
        /* SyncID sanity check */
        if (ipvs->bcfg.syncid != 0 && m2->syncid != ipvs->bcfg.syncid) {
                IP_VS_DBG(7, "BACKUP, Ignoring syncid = %d\n", m2->syncid);
                return;
        }
        /* Handle version 1  message */
        if ((m2->version == SYNC_PROTO_VER) && (m2->reserved == 0)
            && (m2->spare == 0)) {

                msg_end = buffer + sizeof(struct ip_vs_sync_mesg);
                nr_conns = m2->nr_conns;

                for (i=0; i<nr_conns; i++) {
                        union ip_vs_sync_conn *s;
                        unsigned int size;
                        int retc;

                        p = msg_end;
                        if (p + sizeof(s->v4) > buffer+buflen) {
                                IP_VS_ERR_RL("BACKUP, Dropping buffer, too small\n");
                                return;
                        }
                        s = (union ip_vs_sync_conn *)p;
                        size = ntohs(s->v4.ver_size) & SVER_MASK;
                        msg_end = p + size;
                        /* Basic sanity checks */
                        if (msg_end  > buffer+buflen) {
                                IP_VS_ERR_RL("BACKUP, Dropping buffer, msg > buffer\n");
                                return;
                        }
                        if (ntohs(s->v4.ver_size) >> SVER_SHIFT) {
                                IP_VS_ERR_RL("BACKUP, Dropping buffer, Unknown version %d\n",
                                              ntohs(s->v4.ver_size) >> SVER_SHIFT);
                                return;
                        }
                        /* Process a single sync_conn */
                        retc = ip_vs_proc_sync_conn(ipvs, p, msg_end);
                        if (retc < 0) {
                                IP_VS_ERR_RL("BACKUP, Dropping buffer, Err: %d in decoding\n",
                                             retc);
                                return;
                        }
                        /* Make sure we have 32 bit alignment */
                        msg_end = p + ((size + 3) & ~3);
                }
        } else {
                /* Old type of message */
                ip_vs_process_message_v0(ipvs, buffer, buflen);
                return;
        }
}


/*
 *      Setup sndbuf (mode=1) or rcvbuf (mode=0)
 */
static void set_sock_size(struct sock *sk, int mode, int val)
{
        /* setsockopt(sock, SOL_SOCKET, SO_SNDBUF, &val, sizeof(val)); */
        /* setsockopt(sock, SOL_SOCKET, SO_RCVBUF, &val, sizeof(val)); */
        lock_sock(sk);
        if (mode) {
                val = clamp_t(int, val, (SOCK_MIN_SNDBUF + 1) / 2,
                              READ_ONCE(sysctl_wmem_max));
                sk->sk_sndbuf = val * 2;
                sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
        } else {
                val = clamp_t(int, val, (SOCK_MIN_RCVBUF + 1) / 2,
                              READ_ONCE(sysctl_rmem_max));
                sk->sk_rcvbuf = val * 2;
                sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
        }
        release_sock(sk);
}

/*
 *      Setup loopback of outgoing multicasts on a sending socket
 */
static void set_mcast_loop(struct sock *sk, u_char loop)
{
        /* setsockopt(sock, SOL_IP, IP_MULTICAST_LOOP, &loop, sizeof(loop)); */
        inet_assign_bit(MC_LOOP, sk, loop);
#ifdef CONFIG_IP_VS_IPV6
        if (READ_ONCE(sk->sk_family) == AF_INET6) {
                /* IPV6_MULTICAST_LOOP */
                inet6_assign_bit(MC6_LOOP, sk, loop);
        }
#endif
}

/*
 *      Specify TTL for outgoing multicasts on a sending socket
 */
static void set_mcast_ttl(struct sock *sk, u_char ttl)
{
        struct inet_sock *inet = inet_sk(sk);

        /* setsockopt(sock, SOL_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)); */
        lock_sock(sk);
        WRITE_ONCE(inet->mc_ttl, ttl);
#ifdef CONFIG_IP_VS_IPV6
        if (sk->sk_family == AF_INET6) {
                struct ipv6_pinfo *np = inet6_sk(sk);

                /* IPV6_MULTICAST_HOPS */
                WRITE_ONCE(np->mcast_hops, ttl);
        }
#endif
        release_sock(sk);
}

/* Control fragmentation of messages */
static void set_mcast_pmtudisc(struct sock *sk, int val)
{
        struct inet_sock *inet = inet_sk(sk);

        /* setsockopt(sock, SOL_IP, IP_MTU_DISCOVER, &val, sizeof(val)); */
        lock_sock(sk);
        WRITE_ONCE(inet->pmtudisc, val);
#ifdef CONFIG_IP_VS_IPV6
        if (sk->sk_family == AF_INET6) {
                struct ipv6_pinfo *np = inet6_sk(sk);

                /* IPV6_MTU_DISCOVER */
                WRITE_ONCE(np->pmtudisc, val);
        }
#endif
        release_sock(sk);
}

/*
 *      Specifiy default interface for outgoing multicasts
 */
static int set_mcast_if(struct sock *sk, struct net_device *dev)
{
        struct inet_sock *inet = inet_sk(sk);

        if (sk->sk_bound_dev_if && dev->ifindex != sk->sk_bound_dev_if)
                return -EINVAL;

        lock_sock(sk);
        inet->mc_index = dev->ifindex;
        /*  inet->mc_addr  = 0; */
#ifdef CONFIG_IP_VS_IPV6
        if (sk->sk_family == AF_INET6) {
                struct ipv6_pinfo *np = inet6_sk(sk);

                /* IPV6_MULTICAST_IF */
                WRITE_ONCE(np->mcast_oif, dev->ifindex);
        }
#endif
        release_sock(sk);

        return 0;
}


/*
 *      Join a multicast group.
 *      the group is specified by a class D multicast address 224.0.0.0/8
 *      in the in_addr structure passed in as a parameter.
 */
static int
join_mcast_group(struct sock *sk, struct in_addr *addr, struct net_device *dev)
{
        struct ip_mreqn mreq;
        int ret;

        memset(&mreq, 0, sizeof(mreq));
        memcpy(&mreq.imr_multiaddr, addr, sizeof(struct in_addr));

        if (sk->sk_bound_dev_if && dev->ifindex != sk->sk_bound_dev_if)
                return -EINVAL;

        mreq.imr_ifindex = dev->ifindex;

        lock_sock(sk);
        ret = ip_mc_join_group(sk, &mreq);
        release_sock(sk);

        return ret;
}

#ifdef CONFIG_IP_VS_IPV6
static int join_mcast_group6(struct sock *sk, struct in6_addr *addr,
                             struct net_device *dev)
{
        int ret;

        if (sk->sk_bound_dev_if && dev->ifindex != sk->sk_bound_dev_if)
                return -EINVAL;

        lock_sock(sk);
        ret = ipv6_sock_mc_join(sk, dev->ifindex, addr);
        release_sock(sk);

        return ret;
}
#endif

static int bind_mcastif_addr(struct socket *sock, struct net_device *dev)
{
        __be32 addr;
        struct sockaddr_in sin;

        addr = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
        if (!addr)
                pr_err("You probably need to specify IP address on "
                       "multicast interface.\n");

        IP_VS_DBG(7, "binding socket with (%s) %pI4\n",
                  dev->name, &addr);

        /* Now bind the socket with the address of multicast interface */
        sin.sin_family             = AF_INET;
        sin.sin_addr.s_addr  = addr;
        sin.sin_port         = 0;

        return kernel_bind(sock, (struct sockaddr *)&sin, sizeof(sin));
}

static void get_mcast_sockaddr(union ipvs_sockaddr *sa, int *salen,
                               struct ipvs_sync_daemon_cfg *c, int id)
{
        if (AF_INET6 == c->mcast_af) {
                sa->in6 = (struct sockaddr_in6) {
                        .sin6_family = AF_INET6,
                        .sin6_port = htons(c->mcast_port + id),
                };
                sa->in6.sin6_addr = c->mcast_group.in6;
                *salen = sizeof(sa->in6);
        } else {
                sa->in = (struct sockaddr_in) {
                        .sin_family = AF_INET,
                        .sin_port = htons(c->mcast_port + id),
                };
                sa->in.sin_addr = c->mcast_group.in;
                *salen = sizeof(sa->in);
        }
}

/*
 *      Set up sending multicast socket over UDP
 */
static int make_send_sock(struct netns_ipvs *ipvs, int id,
                          struct net_device *dev, struct socket **sock_ret)
{
        /* multicast addr */
        union ipvs_sockaddr mcast_addr;
        struct socket *sock;
        int result, salen;

        /* First create a socket */
        result = sock_create_kern(ipvs->net, ipvs->mcfg.mcast_af, SOCK_DGRAM,
                                  IPPROTO_UDP, &sock);
        if (result < 0) {
                pr_err("Error during creation of socket; terminating\n");
                goto error;
        }
        *sock_ret = sock;
        result = set_mcast_if(sock->sk, dev);
        if (result < 0) {
                pr_err("Error setting outbound mcast interface\n");
                goto error;
        }

        set_mcast_loop(sock->sk, 0);
        set_mcast_ttl(sock->sk, ipvs->mcfg.mcast_ttl);
        /* Allow fragmentation if MTU changes */
        set_mcast_pmtudisc(sock->sk, IP_PMTUDISC_DONT);
        result = sysctl_sync_sock_size(ipvs);
        if (result > 0)
                set_sock_size(sock->sk, 1, result);

        if (AF_INET == ipvs->mcfg.mcast_af)
                result = bind_mcastif_addr(sock, dev);
        else
                result = 0;
        if (result < 0) {
                pr_err("Error binding address of the mcast interface\n");
                goto error;
        }

        get_mcast_sockaddr(&mcast_addr, &salen, &ipvs->mcfg, id);
        result = kernel_connect(sock, (struct sockaddr *)&mcast_addr,
                                salen, 0);
        if (result < 0) {
                pr_err("Error connecting to the multicast addr\n");
                goto error;
        }

        return 0;

error:
        return result;
}


/*
 *      Set up receiving multicast socket over UDP
 */
static int make_receive_sock(struct netns_ipvs *ipvs, int id,
                             struct net_device *dev, struct socket **sock_ret)
{
        /* multicast addr */
        union ipvs_sockaddr mcast_addr;
        struct socket *sock;
        int result, salen;

        /* First create a socket */
        result = sock_create_kern(ipvs->net, ipvs->bcfg.mcast_af, SOCK_DGRAM,
                                  IPPROTO_UDP, &sock);
        if (result < 0) {
                pr_err("Error during creation of socket; terminating\n");
                goto error;
        }
        *sock_ret = sock;
        /* it is equivalent to the REUSEADDR option in user-space */
        sock->sk->sk_reuse = SK_CAN_REUSE;
        result = sysctl_sync_sock_size(ipvs);
        if (result > 0)
                set_sock_size(sock->sk, 0, result);

        get_mcast_sockaddr(&mcast_addr, &salen, &ipvs->bcfg, id);
        sock->sk->sk_bound_dev_if = dev->ifindex;
        result = kernel_bind(sock, (struct sockaddr *)&mcast_addr, salen);
        if (result < 0) {
                pr_err("Error binding to the multicast addr\n");
                goto error;
        }

        /* join the multicast group */
#ifdef CONFIG_IP_VS_IPV6
        if (ipvs->bcfg.mcast_af == AF_INET6)
                result = join_mcast_group6(sock->sk, &mcast_addr.in6.sin6_addr,
                                           dev);
        else
#endif
                result = join_mcast_group(sock->sk, &mcast_addr.in.sin_addr,
                                          dev);
        if (result < 0) {
                pr_err("Error joining to the multicast group\n");
                goto error;
        }

        return 0;

error:
        return result;
}


static int
ip_vs_send_async(struct socket *sock, const char *buffer, const size_t length)
{
        struct msghdr        msg = {.msg_flags = MSG_DONTWAIT|MSG_NOSIGNAL};
        struct kvec        iov;
        int                len;

        iov.iov_base     = (void *)buffer;
        iov.iov_len      = length;

        len = kernel_sendmsg(sock, &msg, &iov, 1, (size_t)(length));

        return len;
}

static int
ip_vs_send_sync_msg(struct socket *sock, struct ip_vs_sync_mesg *msg)
{
        int msize;
        int ret;

        msize = ntohs(msg->size);

        ret = ip_vs_send_async(sock, (char *)msg, msize);
        if (ret >= 0 || ret == -EAGAIN)
                return ret;
        pr_err("ip_vs_send_async error %d\n", ret);
        return 0;
}

static int
ip_vs_receive(struct socket *sock, char *buffer, const size_t buflen)
{
        struct msghdr                msg = {NULL,};
        struct kvec                iov = {buffer, buflen};
        int                        len;

        /* Receive a packet */
        iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, buflen);
        len = sock_recvmsg(sock, &msg, MSG_DONTWAIT);
        if (len < 0)
                return len;

        return len;
}

/* Wakeup the master thread for sending */
static void master_wakeup_work_handler(struct work_struct *work)
{
        struct ipvs_master_sync_state *ms =
                container_of(work, struct ipvs_master_sync_state,
                             master_wakeup_work.work);
        struct netns_ipvs *ipvs = ms->ipvs;

        spin_lock_bh(&ipvs->sync_lock);
        if (ms->sync_queue_len &&
            ms->sync_queue_delay < IPVS_SYNC_WAKEUP_RATE) {
                int id = (int)(ms - ipvs->ms);

                ms->sync_queue_delay = IPVS_SYNC_WAKEUP_RATE;
                wake_up_process(ipvs->master_tinfo[id].task);
        }
        spin_unlock_bh(&ipvs->sync_lock);
}

/* Get next buffer to send */
static inline struct ip_vs_sync_buff *
next_sync_buff(struct netns_ipvs *ipvs, struct ipvs_master_sync_state *ms)
{
        struct ip_vs_sync_buff *sb;

        sb = sb_dequeue(ipvs, ms);
        if (sb)
                return sb;
        /* Do not delay entries in buffer for more than 2 seconds */
        return get_curr_sync_buff(ipvs, ms, IPVS_SYNC_FLUSH_TIME);
}

static int sync_thread_master(void *data)
{
        struct ip_vs_sync_thread_data *tinfo = data;
        struct netns_ipvs *ipvs = tinfo->ipvs;
        struct ipvs_master_sync_state *ms = &ipvs->ms[tinfo->id];
        struct sock *sk = tinfo->sock->sk;
        struct ip_vs_sync_buff *sb;

        pr_info("sync thread started: state = MASTER, mcast_ifn = %s, "
                "syncid = %d, id = %d\n",
                ipvs->mcfg.mcast_ifn, ipvs->mcfg.syncid, tinfo->id);

        for (;;) {
                sb = next_sync_buff(ipvs, ms);
                if (unlikely(kthread_should_stop()))
                        break;
                if (!sb) {
                        schedule_timeout(IPVS_SYNC_CHECK_PERIOD);
                        continue;
                }
                while (ip_vs_send_sync_msg(tinfo->sock, sb->mesg) < 0) {
                        /* (Ab)use interruptible sleep to avoid increasing
                         * the load avg.
                         */
                        __wait_event_interruptible(*sk_sleep(sk),
                                                   sock_writeable(sk) ||
                                                   kthread_should_stop());
                        if (unlikely(kthread_should_stop()))
                                goto done;
                }
                ip_vs_sync_buff_release(sb);
        }

done:
        __set_current_state(TASK_RUNNING);
        if (sb)
                ip_vs_sync_buff_release(sb);

        /* clean up the sync_buff queue */
        while ((sb = sb_dequeue(ipvs, ms)))
                ip_vs_sync_buff_release(sb);
        __set_current_state(TASK_RUNNING);

        /* clean up the current sync_buff */
        sb = get_curr_sync_buff(ipvs, ms, 0);
        if (sb)
                ip_vs_sync_buff_release(sb);

        return 0;
}


static int sync_thread_backup(void *data)
{
        struct ip_vs_sync_thread_data *tinfo = data;
        struct netns_ipvs *ipvs = tinfo->ipvs;
        struct sock *sk = tinfo->sock->sk;
        struct udp_sock *up = udp_sk(sk);
        int len;

        pr_info("sync thread started: state = BACKUP, mcast_ifn = %s, "
                "syncid = %d, id = %d\n",
                ipvs->bcfg.mcast_ifn, ipvs->bcfg.syncid, tinfo->id);

        while (!kthread_should_stop()) {
                wait_event_interruptible(*sk_sleep(sk),
                                         !skb_queue_empty_lockless(&sk->sk_receive_queue) ||
                                         !skb_queue_empty_lockless(&up->reader_queue) ||
                                         kthread_should_stop());

                /* do we have data now? */
                while (!skb_queue_empty_lockless(&sk->sk_receive_queue) ||
                       !skb_queue_empty_lockless(&up->reader_queue)) {
                        len = ip_vs_receive(tinfo->sock, tinfo->buf,
                                        ipvs->bcfg.sync_maxlen);
                        if (len <= 0) {
                                if (len != -EAGAIN)
                                        pr_err("receiving message error\n");
                                break;
                        }

                        ip_vs_process_message(ipvs, tinfo->buf, len);
                }
        }

        return 0;
}


int start_sync_thread(struct netns_ipvs *ipvs, struct ipvs_sync_daemon_cfg *c,
                      int state)
{
        struct ip_vs_sync_thread_data *ti = NULL, *tinfo;
        struct task_struct *task;
        struct net_device *dev;
        char *name;
        int (*threadfn)(void *data);
        int id = 0, count, hlen;
        int result = -ENOMEM;
        u16 mtu, min_mtu;

        IP_VS_DBG(7, "%s(): pid %d\n", __func__, task_pid_nr(current));
        IP_VS_DBG(7, "Each ip_vs_sync_conn entry needs %zd bytes\n",
                  sizeof(struct ip_vs_sync_conn_v0));

        /* increase the module use count */
        if (!ip_vs_use_count_inc())
                return -ENOPROTOOPT;

        /* Do not hold one mutex and then to block on another */
        for (;;) {
                rtnl_lock();
                if (mutex_trylock(&ipvs->sync_mutex))
                        break;
                rtnl_unlock();
                mutex_lock(&ipvs->sync_mutex);
                if (rtnl_trylock())
                        break;
                mutex_unlock(&ipvs->sync_mutex);
        }

        if (!ipvs->sync_state) {
                count = clamp(sysctl_sync_ports(ipvs), 1, IPVS_SYNC_PORTS_MAX);
                ipvs->threads_mask = count - 1;
        } else
                count = ipvs->threads_mask + 1;

        if (c->mcast_af == AF_UNSPEC) {
                c->mcast_af = AF_INET;
                c->mcast_group.ip = cpu_to_be32(IP_VS_SYNC_GROUP);
        }
        if (!c->mcast_port)
                c->mcast_port = IP_VS_SYNC_PORT;
        if (!c->mcast_ttl)
                c->mcast_ttl = 1;

        dev = __dev_get_by_name(ipvs->net, c->mcast_ifn);
        if (!dev) {
                pr_err("Unknown mcast interface: %s\n", c->mcast_ifn);
                result = -ENODEV;
                goto out_early;
        }
        hlen = (AF_INET6 == c->mcast_af) ?
               sizeof(struct ipv6hdr) + sizeof(struct udphdr) :
               sizeof(struct iphdr) + sizeof(struct udphdr);
        mtu = (state == IP_VS_STATE_BACKUP) ?
                  clamp(dev->mtu, 1500U, 65535U) : 1500U;
        min_mtu = (state == IP_VS_STATE_BACKUP) ? 1024 : 1;

        if (c->sync_maxlen)
                c->sync_maxlen = clamp_t(unsigned int,
                                         c->sync_maxlen, min_mtu,
                                         65535 - hlen);
        else
                c->sync_maxlen = mtu - hlen;

        if (state == IP_VS_STATE_MASTER) {
                result = -EEXIST;
                if (ipvs->ms)
                        goto out_early;

                ipvs->mcfg = *c;
                name = "ipvs-m:%d:%d";
                threadfn = sync_thread_master;
        } else if (state == IP_VS_STATE_BACKUP) {
                result = -EEXIST;
                if (ipvs->backup_tinfo)
                        goto out_early;

                ipvs->bcfg = *c;
                name = "ipvs-b:%d:%d";
                threadfn = sync_thread_backup;
        } else {
                result = -EINVAL;
                goto out_early;
        }

        if (state == IP_VS_STATE_MASTER) {
                struct ipvs_master_sync_state *ms;

                result = -ENOMEM;
                ipvs->ms = kcalloc(count, sizeof(ipvs->ms[0]), GFP_KERNEL);
                if (!ipvs->ms)
                        goto out;
                ms = ipvs->ms;
                for (id = 0; id < count; id++, ms++) {
                        INIT_LIST_HEAD(&ms->sync_queue);
                        ms->sync_queue_len = 0;
                        ms->sync_queue_delay = 0;
                        INIT_DELAYED_WORK(&ms->master_wakeup_work,
                                          master_wakeup_work_handler);
                        ms->ipvs = ipvs;
                }
        }
        result = -ENOMEM;
        ti = kcalloc(count, sizeof(struct ip_vs_sync_thread_data),
                     GFP_KERNEL);
        if (!ti)
                goto out;

        for (id = 0; id < count; id++) {
                tinfo = &ti[id];
                tinfo->ipvs = ipvs;
                if (state == IP_VS_STATE_BACKUP) {
                        result = -ENOMEM;
                        tinfo->buf = kmalloc(ipvs->bcfg.sync_maxlen,
                                             GFP_KERNEL);
                        if (!tinfo->buf)
                                goto out;
                }
                tinfo->id = id;
                if (state == IP_VS_STATE_MASTER)
                        result = make_send_sock(ipvs, id, dev, &tinfo->sock);
                else
                        result = make_receive_sock(ipvs, id, dev, &tinfo->sock);
                if (result < 0)
                        goto out;

                task = kthread_run(threadfn, tinfo, name, ipvs->gen, id);
                if (IS_ERR(task)) {
                        result = PTR_ERR(task);
                        goto out;
                }
                tinfo->task = task;
        }

        /* mark as active */

        if (state == IP_VS_STATE_MASTER)
                ipvs->master_tinfo = ti;
        else
                ipvs->backup_tinfo = ti;
        spin_lock_bh(&ipvs->sync_buff_lock);
        ipvs->sync_state |= state;
        spin_unlock_bh(&ipvs->sync_buff_lock);

        mutex_unlock(&ipvs->sync_mutex);
        rtnl_unlock();

        return 0;

out:
        /* We do not need RTNL lock anymore, release it here so that
         * sock_release below can use rtnl_lock to leave the mcast group.
         */
        rtnl_unlock();
        id = min(id, count - 1);
        if (ti) {
                for (tinfo = ti + id; tinfo >= ti; tinfo--) {
                        if (tinfo->task)
                                kthread_stop(tinfo->task);
                }
        }
        if (!(ipvs->sync_state & IP_VS_STATE_MASTER)) {
                kfree(ipvs->ms);
                ipvs->ms = NULL;
        }
        mutex_unlock(&ipvs->sync_mutex);

        /* No more mutexes, release socks */
        if (ti) {
                for (tinfo = ti + id; tinfo >= ti; tinfo--) {
                        if (tinfo->sock)
                                sock_release(tinfo->sock);
                        kfree(tinfo->buf);
                }
                kfree(ti);
        }

        /* decrease the module use count */
        ip_vs_use_count_dec();
        return result;

out_early:
        mutex_unlock(&ipvs->sync_mutex);
        rtnl_unlock();

        /* decrease the module use count */
        ip_vs_use_count_dec();
        return result;
}


int stop_sync_thread(struct netns_ipvs *ipvs, int state)
{
        struct ip_vs_sync_thread_data *ti, *tinfo;
        int id;
        int retc = -EINVAL;

        IP_VS_DBG(7, "%s(): pid %d\n", __func__, task_pid_nr(current));

        mutex_lock(&ipvs->sync_mutex);
        if (state == IP_VS_STATE_MASTER) {
                retc = -ESRCH;
                if (!ipvs->ms)
                        goto err;
                ti = ipvs->master_tinfo;

                /*
                 * The lock synchronizes with sb_queue_tail(), so that we don't
                 * add sync buffers to the queue, when we are already in
                 * progress of stopping the master sync daemon.
                 */

                spin_lock_bh(&ipvs->sync_buff_lock);
                spin_lock(&ipvs->sync_lock);
                ipvs->sync_state &= ~IP_VS_STATE_MASTER;
                spin_unlock(&ipvs->sync_lock);
                spin_unlock_bh(&ipvs->sync_buff_lock);

                retc = 0;
                for (id = ipvs->threads_mask; id >= 0; id--) {
                        struct ipvs_master_sync_state *ms = &ipvs->ms[id];
                        int ret;

                        tinfo = &ti[id];
                        pr_info("stopping master sync thread %d ...\n",
                                task_pid_nr(tinfo->task));
                        cancel_delayed_work_sync(&ms->master_wakeup_work);
                        ret = kthread_stop(tinfo->task);
                        if (retc >= 0)
                                retc = ret;
                }
                kfree(ipvs->ms);
                ipvs->ms = NULL;
                ipvs->master_tinfo = NULL;
        } else if (state == IP_VS_STATE_BACKUP) {
                retc = -ESRCH;
                if (!ipvs->backup_tinfo)
                        goto err;
                ti = ipvs->backup_tinfo;

                ipvs->sync_state &= ~IP_VS_STATE_BACKUP;
                retc = 0;
                for (id = ipvs->threads_mask; id >= 0; id--) {
                        int ret;

                        tinfo = &ti[id];
                        pr_info("stopping backup sync thread %d ...\n",
                                task_pid_nr(tinfo->task));
                        ret = kthread_stop(tinfo->task);
                        if (retc >= 0)
                                retc = ret;
                }
                ipvs->backup_tinfo = NULL;
        } else {
                goto err;
        }
        id = ipvs->threads_mask;
        mutex_unlock(&ipvs->sync_mutex);

        /* No more mutexes, release socks */
        for (tinfo = ti + id; tinfo >= ti; tinfo--) {
                if (tinfo->sock)
                        sock_release(tinfo->sock);
                kfree(tinfo->buf);
        }
        kfree(ti);

        /* decrease the module use count */
        ip_vs_use_count_dec();
        return retc;

err:
        mutex_unlock(&ipvs->sync_mutex);
        return retc;
}

/*
 * Initialize data struct for each netns
 */
int __net_init ip_vs_sync_net_init(struct netns_ipvs *ipvs)
{
        __mutex_init(&ipvs->sync_mutex, "ipvs->sync_mutex", &__ipvs_sync_key);
        spin_lock_init(&ipvs->sync_lock);
        spin_lock_init(&ipvs->sync_buff_lock);
        return 0;
}

void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs)
{
        int retc;

        retc = stop_sync_thread(ipvs, IP_VS_STATE_MASTER);
        if (retc && retc != -ESRCH)
                pr_err("Failed to stop Master Daemon\n");

        retc = stop_sync_thread(ipvs, IP_VS_STATE_BACKUP);
        if (retc && retc != -ESRCH)
                pr_err("Failed to stop Backup Daemon\n");
}

































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * MACsec netdev header, used for h/w accelerated implementations.
 *
 * Copyright (c) 2015 Sabrina Dubroca <sd@queasysnail.net>
 */
#ifndef _NET_MACSEC_H_
#define _NET_MACSEC_H_

#include <linux/u64_stats_sync.h>
#include <linux/if_vlan.h>
#include <uapi/linux/if_link.h>
#include <uapi/linux/if_macsec.h>

#define MACSEC_DEFAULT_PN_LEN 4
#define MACSEC_XPN_PN_LEN 8

#define MACSEC_NUM_AN 4 /* 2 bits for the association number */

#define MACSEC_SCI_LEN 8
#define MACSEC_PORT_ES (htons(0x0001))

#define MACSEC_TCI_VERSION 0x80
#define MACSEC_TCI_ES      0x40 /* end station */
#define MACSEC_TCI_SC      0x20 /* SCI present */
#define MACSEC_TCI_SCB     0x10 /* epon */
#define MACSEC_TCI_E       0x08 /* encryption */
#define MACSEC_TCI_C       0x04 /* changed text */
#define MACSEC_AN_MASK     0x03 /* association number */
#define MACSEC_TCI_CONFID  (MACSEC_TCI_E | MACSEC_TCI_C)

#define MACSEC_DEFAULT_ICV_LEN 16

typedef u64 __bitwise sci_t;
typedef u32 __bitwise ssci_t;

struct metadata_dst;

typedef union salt {
        struct {
                ssci_t ssci;
                __be64 pn;
        } __packed;
        u8 bytes[MACSEC_SALT_LEN];
} __packed salt_t;

typedef union pn {
        struct {
#if defined(__LITTLE_ENDIAN_BITFIELD)
                u32 lower;
                u32 upper;
#elif defined(__BIG_ENDIAN_BITFIELD)
                u32 upper;
                u32 lower;
#else
#error        "Please fix <asm/byteorder.h>"
#endif
        };
        u64 full64;
} pn_t;

/**
 * struct macsec_key - SA key
 * @id: user-provided key identifier
 * @tfm: crypto struct, key storage
 * @salt: salt used to generate IV in XPN cipher suites
 */
struct macsec_key {
        u8 id[MACSEC_KEYID_LEN];
        struct crypto_aead *tfm;
        salt_t salt;
};

struct macsec_rx_sc_stats {
        __u64 InOctetsValidated;
        __u64 InOctetsDecrypted;
        __u64 InPktsUnchecked;
        __u64 InPktsDelayed;
        __u64 InPktsOK;
        __u64 InPktsInvalid;
        __u64 InPktsLate;
        __u64 InPktsNotValid;
        __u64 InPktsNotUsingSA;
        __u64 InPktsUnusedSA;
};

struct macsec_rx_sa_stats {
        __u32 InPktsOK;
        __u32 InPktsInvalid;
        __u32 InPktsNotValid;
        __u32 InPktsNotUsingSA;
        __u32 InPktsUnusedSA;
};

struct macsec_tx_sa_stats {
        __u32 OutPktsProtected;
        __u32 OutPktsEncrypted;
};

struct macsec_tx_sc_stats {
        __u64 OutPktsProtected;
        __u64 OutPktsEncrypted;
        __u64 OutOctetsProtected;
        __u64 OutOctetsEncrypted;
};

struct macsec_dev_stats {
        __u64 OutPktsUntagged;
        __u64 InPktsUntagged;
        __u64 OutPktsTooLong;
        __u64 InPktsNoTag;
        __u64 InPktsBadTag;
        __u64 InPktsUnknownSCI;
        __u64 InPktsNoSCI;
        __u64 InPktsOverrun;
};

/**
 * struct macsec_rx_sa - receive secure association
 * @active:
 * @next_pn: packet number expected for the next packet
 * @lock: protects next_pn manipulations
 * @key: key structure
 * @ssci: short secure channel identifier
 * @stats: per-SA stats
 */
struct macsec_rx_sa {
        struct macsec_key key;
        ssci_t ssci;
        spinlock_t lock;
        union {
                pn_t next_pn_halves;
                u64 next_pn;
        };
        refcount_t refcnt;
        bool active;
        struct macsec_rx_sa_stats __percpu *stats;
        struct macsec_rx_sc *sc;
        struct rcu_head rcu;
};

struct pcpu_rx_sc_stats {
        struct macsec_rx_sc_stats stats;
        struct u64_stats_sync syncp;
};

struct pcpu_tx_sc_stats {
        struct macsec_tx_sc_stats stats;
        struct u64_stats_sync syncp;
};

/**
 * struct macsec_rx_sc - receive secure channel
 * @sci: secure channel identifier for this SC
 * @active: channel is active
 * @sa: array of secure associations
 * @stats: per-SC stats
 */
struct macsec_rx_sc {
        struct macsec_rx_sc __rcu *next;
        sci_t sci;
        bool active;
        struct macsec_rx_sa __rcu *sa[MACSEC_NUM_AN];
        struct pcpu_rx_sc_stats __percpu *stats;
        refcount_t refcnt;
        struct rcu_head rcu_head;
};

/**
 * struct macsec_tx_sa - transmit secure association
 * @active:
 * @next_pn: packet number to use for the next packet
 * @lock: protects next_pn manipulations
 * @key: key structure
 * @ssci: short secure channel identifier
 * @stats: per-SA stats
 */
struct macsec_tx_sa {
        struct macsec_key key;
        ssci_t ssci;
        spinlock_t lock;
        union {
                pn_t next_pn_halves;
                u64 next_pn;
        };
        refcount_t refcnt;
        bool active;
        struct macsec_tx_sa_stats __percpu *stats;
        struct rcu_head rcu;
};

/**
 * struct macsec_tx_sc - transmit secure channel
 * @active:
 * @encoding_sa: association number of the SA currently in use
 * @encrypt: encrypt packets on transmit, or authenticate only
 * @send_sci: always include the SCI in the SecTAG
 * @end_station:
 * @scb: single copy broadcast flag
 * @sa: array of secure associations
 * @stats: stats for this TXSC
 * @md_dst: MACsec offload metadata dst
 */
struct macsec_tx_sc {
        bool active;
        u8 encoding_sa;
        bool encrypt;
        bool send_sci;
        bool end_station;
        bool scb;
        struct macsec_tx_sa __rcu *sa[MACSEC_NUM_AN];
        struct pcpu_tx_sc_stats __percpu *stats;
        struct metadata_dst *md_dst;
};

/**
 * struct macsec_secy - MACsec Security Entity
 * @netdev: netdevice for this SecY
 * @n_rx_sc: number of receive secure channels configured on this SecY
 * @sci: secure channel identifier used for tx
 * @key_len: length of keys used by the cipher suite
 * @icv_len: length of ICV used by the cipher suite
 * @validate_frames: validation mode
 * @xpn: enable XPN for this SecY
 * @operational: MAC_Operational flag
 * @protect_frames: enable protection for this SecY
 * @replay_protect: enable packet number checks on receive
 * @replay_window: size of the replay window
 * @tx_sc: transmit secure channel
 * @rx_sc: linked list of receive secure channels
 */
struct macsec_secy {
        struct net_device *netdev;
        unsigned int n_rx_sc;
        sci_t sci;
        u16 key_len;
        u16 icv_len;
        enum macsec_validation_type validate_frames;
        bool xpn;
        bool operational;
        bool protect_frames;
        bool replay_protect;
        u32 replay_window;
        struct macsec_tx_sc tx_sc;
        struct macsec_rx_sc __rcu *rx_sc;
};

/**
 * struct macsec_context - MACsec context for hardware offloading
 * @netdev: a valid pointer to a struct net_device if @offload ==
 *        MACSEC_OFFLOAD_MAC
 * @phydev: a valid pointer to a struct phy_device if @offload ==
 *        MACSEC_OFFLOAD_PHY
 * @offload: MACsec offload status
 * @secy: pointer to a MACsec SecY
 * @rx_sc: pointer to a RX SC
 * @update_pn: when updating the SA, update the next PN
 * @assoc_num: association number of the target SA
 * @key: key of the target SA
 * @rx_sa: pointer to an RX SA if a RX SA is added/updated/removed
 * @tx_sa: pointer to an TX SA if a TX SA is added/updated/removed
 * @tx_sc_stats: pointer to TX SC stats structure
 * @tx_sa_stats: pointer to TX SA stats structure
 * @rx_sc_stats: pointer to RX SC stats structure
 * @rx_sa_stats: pointer to RX SA stats structure
 * @dev_stats: pointer to dev stats structure
 */
struct macsec_context {
        union {
                struct net_device *netdev;
                struct phy_device *phydev;
        };
        enum macsec_offload offload;

        struct macsec_secy *secy;
        struct macsec_rx_sc *rx_sc;
        struct {
                bool update_pn;
                unsigned char assoc_num;
                u8 key[MACSEC_MAX_KEY_LEN];
                union {
                        struct macsec_rx_sa *rx_sa;
                        struct macsec_tx_sa *tx_sa;
                };
        } sa;
        union {
                struct macsec_tx_sc_stats *tx_sc_stats;
                struct macsec_tx_sa_stats *tx_sa_stats;
                struct macsec_rx_sc_stats *rx_sc_stats;
                struct macsec_rx_sa_stats *rx_sa_stats;
                struct macsec_dev_stats  *dev_stats;
        } stats;
};

/**
 * struct macsec_ops - MACsec offloading operations
 * @mdo_dev_open: called when the MACsec interface transitions to the up state
 * @mdo_dev_stop: called when the MACsec interface transitions to the down
 *        state
 * @mdo_add_secy: called when a new SecY is added
 * @mdo_upd_secy: called when the SecY flags are changed or the MAC address of
 *        the MACsec interface is changed
 * @mdo_del_secy: called when the hw offload is disabled or the MACsec
 *        interface is removed
 * @mdo_add_rxsc: called when a new RX SC is added
 * @mdo_upd_rxsc: called when a certain RX SC is updated
 * @mdo_del_rxsc: called when a certain RX SC is removed
 * @mdo_add_rxsa: called when a new RX SA is added
 * @mdo_upd_rxsa: called when a certain RX SA is updated
 * @mdo_del_rxsa: called when a certain RX SA is removed
 * @mdo_add_txsa: called when a new TX SA is added
 * @mdo_upd_txsa: called when a certain TX SA is updated
 * @mdo_del_txsa: called when a certain TX SA is removed
 * @mdo_get_dev_stats: called when dev stats are read
 * @mdo_get_tx_sc_stats: called when TX SC stats are read
 * @mdo_get_tx_sa_stats: called when TX SA stats are read
 * @mdo_get_rx_sc_stats: called when RX SC stats are read
 * @mdo_get_rx_sa_stats: called when RX SA stats are read
 * @mdo_insert_tx_tag: called to insert the TX tag
 * @needed_headroom: number of bytes reserved at the beginning of the sk_buff
 *        for the TX tag
 * @needed_tailroom: number of bytes reserved at the end of the sk_buff for the
 *        TX tag
 * @rx_uses_md_dst: whether MACsec device offload supports sk_buff md_dst
 */
struct macsec_ops {
        /* Device wide */
        int (*mdo_dev_open)(struct macsec_context *ctx);
        int (*mdo_dev_stop)(struct macsec_context *ctx);
        /* SecY */
        int (*mdo_add_secy)(struct macsec_context *ctx);
        int (*mdo_upd_secy)(struct macsec_context *ctx);
        int (*mdo_del_secy)(struct macsec_context *ctx);
        /* Security channels */
        int (*mdo_add_rxsc)(struct macsec_context *ctx);
        int (*mdo_upd_rxsc)(struct macsec_context *ctx);
        int (*mdo_del_rxsc)(struct macsec_context *ctx);
        /* Security associations */
        int (*mdo_add_rxsa)(struct macsec_context *ctx);
        int (*mdo_upd_rxsa)(struct macsec_context *ctx);
        int (*mdo_del_rxsa)(struct macsec_context *ctx);
        int (*mdo_add_txsa)(struct macsec_context *ctx);
        int (*mdo_upd_txsa)(struct macsec_context *ctx);
        int (*mdo_del_txsa)(struct macsec_context *ctx);
        /* Statistics */
        int (*mdo_get_dev_stats)(struct macsec_context *ctx);
        int (*mdo_get_tx_sc_stats)(struct macsec_context *ctx);
        int (*mdo_get_tx_sa_stats)(struct macsec_context *ctx);
        int (*mdo_get_rx_sc_stats)(struct macsec_context *ctx);
        int (*mdo_get_rx_sa_stats)(struct macsec_context *ctx);
        /* Offload tag */
        int (*mdo_insert_tx_tag)(struct phy_device *phydev,
                                 struct sk_buff *skb);
        unsigned int needed_headroom;
        unsigned int needed_tailroom;
        bool rx_uses_md_dst;
};

void macsec_pn_wrapped(struct macsec_secy *secy, struct macsec_tx_sa *tx_sa);
static inline bool macsec_send_sci(const struct macsec_secy *secy)
{
        const struct macsec_tx_sc *tx_sc = &secy->tx_sc;

        return tx_sc->send_sci ||
                (secy->n_rx_sc > 1 && !tx_sc->end_station && !tx_sc->scb);
}
struct net_device *macsec_get_real_dev(const struct net_device *dev);
bool macsec_netdev_is_offloaded(struct net_device *dev);

static inline void *macsec_netdev_priv(const struct net_device *dev)
{
#if IS_ENABLED(CONFIG_VLAN_8021Q)
        if (is_vlan_dev(dev))
                return netdev_priv(vlan_dev_priv(dev)->real_dev);
#endif
        return netdev_priv(dev);
}

static inline u64 sci_to_cpu(sci_t sci)
{
        return be64_to_cpu((__force __be64)sci);
}

#endif /* _NET_MACSEC_H_ */



































    8 





























    8 

    8 



















   68 





   69 



   69 



   68 




   68 
   67 


   67 
   69 

   69 

   67 


































   42 

   40 

   40 















































































































   16 





   16 
   15 







   14 
    9 

   14 

   13 




   12 
   12 
   12 
   12 
   12 






    2 

    3 



   20 


   20 










   18 

   20 
   20 

   12 




   11 

    8 
















   26 





   28 
   27 


   23 
   19 

   23 

   22 




   22 
   22 
   21 
   22 
   22 







    1 

    2 



   32 


   32 










   32 

   30 
   32 

   20 





   21 

   10 






































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/readdir.c
 *
 *  Copyright (C) 1995  Linus Torvalds
 */

#include <linux/stddef.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/time.h>
#include <linux/mm.h>
#include <linux/errno.h>
#include <linux/stat.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/fsnotify.h>
#include <linux/dirent.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/unistd.h>
#include <linux/compat.h>
#include <linux/uaccess.h>

/*
 * Some filesystems were never converted to '->iterate_shared()'
 * and their directory iterators want the inode lock held for
 * writing. This wrapper allows for converting from the shared
 * semantics to the exclusive inode use.
 */
int wrap_directory_iterator(struct file *file,
                            struct dir_context *ctx,
                            int (*iter)(struct file *, struct dir_context *))
{
        struct inode *inode = file_inode(file);
        int ret;

        /*
         * We'd love to have an 'inode_upgrade_trylock()' operation,
         * see the comment in mmap_upgrade_trylock() in mm/memory.c.
         *
         * But considering this is for "filesystems that never got
         * converted", it really doesn't matter.
         *
         * Also note that since we have to return with the lock held
         * for reading, we can't use the "killable()" locking here,
         * since we do need to get the lock even if we're dying.
         *
         * We could do the write part killably and then get the read
         * lock unconditionally if it mattered, but see above on why
         * this does the very simplistic conversion.
         */
        up_read(&inode->i_rwsem);
        down_write(&inode->i_rwsem);

        /*
         * Since we dropped the inode lock, we should do the
         * DEADDIR test again. See 'iterate_dir()' below.
         *
         * Note that we don't need to re-do the f_pos games,
         * since the file must be locked wrt f_pos anyway.
         */
        ret = -ENOENT;
        if (!IS_DEADDIR(inode))
                ret = iter(file, ctx);

        downgrade_write(&inode->i_rwsem);
        return ret;
}
EXPORT_SYMBOL(wrap_directory_iterator);

/*
 * Note the "unsafe_put_user()" semantics: we goto a
 * label for errors.
 */
#define unsafe_copy_dirent_name(_dst, _src, _len, label) do {        \
        char __user *dst = (_dst);                                \
        const char *src = (_src);                                \
        size_t len = (_len);                                        \
        unsafe_put_user(0, dst+len, label);                        \
        unsafe_copy_to_user(dst, src, len, label);                \
} while (0)


int iterate_dir(struct file *file, struct dir_context *ctx)
{
        struct inode *inode = file_inode(file);
        int res = -ENOTDIR;

        if (!file->f_op->iterate_shared)
                goto out;

        res = security_file_permission(file, MAY_READ);
        if (res)
                goto out;

        res = fsnotify_file_perm(file, MAY_READ);
        if (res)
                goto out;

        res = down_read_killable(&inode->i_rwsem);
        if (res)
                goto out;

        res = -ENOENT;
        if (!IS_DEADDIR(inode)) {
                ctx->pos = file->f_pos;
                res = file->f_op->iterate_shared(file, ctx);
                file->f_pos = ctx->pos;
                fsnotify_access(file);
                file_accessed(file);
        }
        inode_unlock_shared(inode);
out:
        return res;
}
EXPORT_SYMBOL(iterate_dir);

/*
 * POSIX says that a dirent name cannot contain NULL or a '/'.
 *
 * It's not 100% clear what we should really do in this case.
 * The filesystem is clearly corrupted, but returning a hard
 * error means that you now don't see any of the other names
 * either, so that isn't a perfect alternative.
 *
 * And if you return an error, what error do you use? Several
 * filesystems seem to have decided on EUCLEAN being the error
 * code for EFSCORRUPTED, and that may be the error to use. Or
 * just EIO, which is perhaps more obvious to users.
 *
 * In order to see the other file names in the directory, the
 * caller might want to make this a "soft" error: skip the
 * entry, and return the error at the end instead.
 *
 * Note that this should likely do a "memchr(name, 0, len)"
 * check too, since that would be filesystem corruption as
 * well. However, that case can't actually confuse user space,
 * which has to do a strlen() on the name anyway to find the
 * filename length, and the above "soft error" worry means
 * that it's probably better left alone until we have that
 * issue clarified.
 *
 * Note the PATH_MAX check - it's arbitrary but the real
 * kernel limit on a possible path component, not NAME_MAX,
 * which is the technical standard limit.
 */
static int verify_dirent_name(const char *name, int len)
{
        if (len <= 0 || len >= PATH_MAX)
                return -EIO;
        if (memchr(name, '/', len))
                return -EIO;
        return 0;
}

/*
 * Traditional linux readdir() handling..
 *
 * "count=1" is a special case, meaning that the buffer is one
 * dirent-structure in size and that the code can't handle more
 * anyway. Thus the special "fillonedir()" function for that
 * case (the low-level handlers don't need to care about this).
 */

#ifdef __ARCH_WANT_OLD_READDIR

struct old_linux_dirent {
        unsigned long        d_ino;
        unsigned long        d_offset;
        unsigned short        d_namlen;
        char                d_name[];
};

struct readdir_callback {
        struct dir_context ctx;
        struct old_linux_dirent __user * dirent;
        int result;
};

static bool fillonedir(struct dir_context *ctx, const char *name, int namlen,
                      loff_t offset, u64 ino, unsigned int d_type)
{
        struct readdir_callback *buf =
                container_of(ctx, struct readdir_callback, ctx);
        struct old_linux_dirent __user * dirent;
        unsigned long d_ino;

        if (buf->result)
                return false;
        buf->result = verify_dirent_name(name, namlen);
        if (buf->result)
                return false;
        d_ino = ino;
        if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) {
                buf->result = -EOVERFLOW;
                return false;
        }
        buf->result++;
        dirent = buf->dirent;
        if (!user_write_access_begin(dirent,
                        (unsigned long)(dirent->d_name + namlen + 1) -
                                (unsigned long)dirent))
                goto efault;
        unsafe_put_user(d_ino, &dirent->d_ino, efault_end);
        unsafe_put_user(offset, &dirent->d_offset, efault_end);
        unsafe_put_user(namlen, &dirent->d_namlen, efault_end);
        unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault_end);
        user_write_access_end();
        return true;
efault_end:
        user_write_access_end();
efault:
        buf->result = -EFAULT;
        return false;
}

SYSCALL_DEFINE3(old_readdir, unsigned int, fd,
                struct old_linux_dirent __user *, dirent, unsigned int, count)
{
        int error;
        CLASS(fd_pos, f)(fd);
        struct readdir_callback buf = {
                .ctx.actor = fillonedir,
                .dirent = dirent
        };

        if (fd_empty(f))
                return -EBADF;

        error = iterate_dir(fd_file(f), &buf.ctx);
        if (buf.result)
                error = buf.result;

        return error;
}

#endif /* __ARCH_WANT_OLD_READDIR */

/*
 * New, all-improved, singing, dancing, iBCS2-compliant getdents()
 * interface. 
 */
struct linux_dirent {
        unsigned long        d_ino;
        unsigned long        d_off;
        unsigned short        d_reclen;
        char                d_name[];
};

struct getdents_callback {
        struct dir_context ctx;
        struct linux_dirent __user * current_dir;
        int prev_reclen;
        int count;
        int error;
};

static bool filldir(struct dir_context *ctx, const char *name, int namlen,
                   loff_t offset, u64 ino, unsigned int d_type)
{
        struct linux_dirent __user *dirent, *prev;
        struct getdents_callback *buf =
                container_of(ctx, struct getdents_callback, ctx);
        unsigned long d_ino;
        int reclen = ALIGN(offsetof(struct linux_dirent, d_name) + namlen + 2,
                sizeof(long));
        int prev_reclen;

        buf->error = verify_dirent_name(name, namlen);
        if (unlikely(buf->error))
                return false;
        buf->error = -EINVAL;        /* only used if we fail.. */
        if (reclen > buf->count)
                return false;
        d_ino = ino;
        if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) {
                buf->error = -EOVERFLOW;
                return false;
        }
        prev_reclen = buf->prev_reclen;
        if (prev_reclen && signal_pending(current))
                return false;
        dirent = buf->current_dir;
        prev = (void __user *) dirent - prev_reclen;
        if (!user_write_access_begin(prev, reclen + prev_reclen))
                goto efault;

        /* This might be 'dirent->d_off', but if so it will get overwritten */
        unsafe_put_user(offset, &prev->d_off, efault_end);
        unsafe_put_user(d_ino, &dirent->d_ino, efault_end);
        unsafe_put_user(reclen, &dirent->d_reclen, efault_end);
        unsafe_put_user(d_type, (char __user *) dirent + reclen - 1, efault_end);
        unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault_end);
        user_write_access_end();

        buf->current_dir = (void __user *)dirent + reclen;
        buf->prev_reclen = reclen;
        buf->count -= reclen;
        return true;
efault_end:
        user_write_access_end();
efault:
        buf->error = -EFAULT;
        return false;
}

SYSCALL_DEFINE3(getdents, unsigned int, fd,
                struct linux_dirent __user *, dirent, unsigned int, count)
{
        CLASS(fd_pos, f)(fd);
        struct getdents_callback buf = {
                .ctx.actor = filldir,
                .count = count,
                .current_dir = dirent
        };
        int error;

        if (fd_empty(f))
                return -EBADF;

        error = iterate_dir(fd_file(f), &buf.ctx);
        if (error >= 0)
                error = buf.error;
        if (buf.prev_reclen) {
                struct linux_dirent __user * lastdirent;
                lastdirent = (void __user *)buf.current_dir - buf.prev_reclen;

                if (put_user(buf.ctx.pos, &lastdirent->d_off))
                        error = -EFAULT;
                else
                        error = count - buf.count;
        }
        return error;
}

struct getdents_callback64 {
        struct dir_context ctx;
        struct linux_dirent64 __user * current_dir;
        int prev_reclen;
        int count;
        int error;
};

static bool filldir64(struct dir_context *ctx, const char *name, int namlen,
                     loff_t offset, u64 ino, unsigned int d_type)
{
        struct linux_dirent64 __user *dirent, *prev;
        struct getdents_callback64 *buf =
                container_of(ctx, struct getdents_callback64, ctx);
        int reclen = ALIGN(offsetof(struct linux_dirent64, d_name) + namlen + 1,
                sizeof(u64));
        int prev_reclen;

        buf->error = verify_dirent_name(name, namlen);
        if (unlikely(buf->error))
                return false;
        buf->error = -EINVAL;        /* only used if we fail.. */
        if (reclen > buf->count)
                return false;
        prev_reclen = buf->prev_reclen;
        if (prev_reclen && signal_pending(current))
                return false;
        dirent = buf->current_dir;
        prev = (void __user *)dirent - prev_reclen;
        if (!user_write_access_begin(prev, reclen + prev_reclen))
                goto efault;

        /* This might be 'dirent->d_off', but if so it will get overwritten */
        unsafe_put_user(offset, &prev->d_off, efault_end);
        unsafe_put_user(ino, &dirent->d_ino, efault_end);
        unsafe_put_user(reclen, &dirent->d_reclen, efault_end);
        unsafe_put_user(d_type, &dirent->d_type, efault_end);
        unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault_end);
        user_write_access_end();

        buf->prev_reclen = reclen;
        buf->current_dir = (void __user *)dirent + reclen;
        buf->count -= reclen;
        return true;

efault_end:
        user_write_access_end();
efault:
        buf->error = -EFAULT;
        return false;
}

SYSCALL_DEFINE3(getdents64, unsigned int, fd,
                struct linux_dirent64 __user *, dirent, unsigned int, count)
{
        CLASS(fd_pos, f)(fd);
        struct getdents_callback64 buf = {
                .ctx.actor = filldir64,
                .count = count,
                .current_dir = dirent
        };
        int error;

        if (fd_empty(f))
                return -EBADF;

        error = iterate_dir(fd_file(f), &buf.ctx);
        if (error >= 0)
                error = buf.error;
        if (buf.prev_reclen) {
                struct linux_dirent64 __user * lastdirent;
                typeof(lastdirent->d_off) d_off = buf.ctx.pos;

                lastdirent = (void __user *) buf.current_dir - buf.prev_reclen;
                if (put_user(d_off, &lastdirent->d_off))
                        error = -EFAULT;
                else
                        error = count - buf.count;
        }
        return error;
}

#ifdef CONFIG_COMPAT
struct compat_old_linux_dirent {
        compat_ulong_t        d_ino;
        compat_ulong_t        d_offset;
        unsigned short        d_namlen;
        char                d_name[];
};

struct compat_readdir_callback {
        struct dir_context ctx;
        struct compat_old_linux_dirent __user *dirent;
        int result;
};

static bool compat_fillonedir(struct dir_context *ctx, const char *name,
                             int namlen, loff_t offset, u64 ino,
                             unsigned int d_type)
{
        struct compat_readdir_callback *buf =
                container_of(ctx, struct compat_readdir_callback, ctx);
        struct compat_old_linux_dirent __user *dirent;
        compat_ulong_t d_ino;

        if (buf->result)
                return false;
        buf->result = verify_dirent_name(name, namlen);
        if (buf->result)
                return false;
        d_ino = ino;
        if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) {
                buf->result = -EOVERFLOW;
                return false;
        }
        buf->result++;
        dirent = buf->dirent;
        if (!user_write_access_begin(dirent,
                        (unsigned long)(dirent->d_name + namlen + 1) -
                                (unsigned long)dirent))
                goto efault;
        unsafe_put_user(d_ino, &dirent->d_ino, efault_end);
        unsafe_put_user(offset, &dirent->d_offset, efault_end);
        unsafe_put_user(namlen, &dirent->d_namlen, efault_end);
        unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault_end);
        user_write_access_end();
        return true;
efault_end:
        user_write_access_end();
efault:
        buf->result = -EFAULT;
        return false;
}

COMPAT_SYSCALL_DEFINE3(old_readdir, unsigned int, fd,
                struct compat_old_linux_dirent __user *, dirent, unsigned int, count)
{
        int error;
        CLASS(fd_pos, f)(fd);
        struct compat_readdir_callback buf = {
                .ctx.actor = compat_fillonedir,
                .dirent = dirent
        };

        if (fd_empty(f))
                return -EBADF;

        error = iterate_dir(fd_file(f), &buf.ctx);
        if (buf.result)
                error = buf.result;

        return error;
}

struct compat_linux_dirent {
        compat_ulong_t        d_ino;
        compat_ulong_t        d_off;
        unsigned short        d_reclen;
        char                d_name[];
};

struct compat_getdents_callback {
        struct dir_context ctx;
        struct compat_linux_dirent __user *current_dir;
        int prev_reclen;
        int count;
        int error;
};

static bool compat_filldir(struct dir_context *ctx, const char *name, int namlen,
                loff_t offset, u64 ino, unsigned int d_type)
{
        struct compat_linux_dirent __user *dirent, *prev;
        struct compat_getdents_callback *buf =
                container_of(ctx, struct compat_getdents_callback, ctx);
        compat_ulong_t d_ino;
        int reclen = ALIGN(offsetof(struct compat_linux_dirent, d_name) +
                namlen + 2, sizeof(compat_long_t));
        int prev_reclen;

        buf->error = verify_dirent_name(name, namlen);
        if (unlikely(buf->error))
                return false;
        buf->error = -EINVAL;        /* only used if we fail.. */
        if (reclen > buf->count)
                return false;
        d_ino = ino;
        if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) {
                buf->error = -EOVERFLOW;
                return false;
        }
        prev_reclen = buf->prev_reclen;
        if (prev_reclen && signal_pending(current))
                return false;
        dirent = buf->current_dir;
        prev = (void __user *) dirent - prev_reclen;
        if (!user_write_access_begin(prev, reclen + prev_reclen))
                goto efault;

        unsafe_put_user(offset, &prev->d_off, efault_end);
        unsafe_put_user(d_ino, &dirent->d_ino, efault_end);
        unsafe_put_user(reclen, &dirent->d_reclen, efault_end);
        unsafe_put_user(d_type, (char __user *) dirent + reclen - 1, efault_end);
        unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault_end);
        user_write_access_end();

        buf->prev_reclen = reclen;
        buf->current_dir = (void __user *)dirent + reclen;
        buf->count -= reclen;
        return true;
efault_end:
        user_write_access_end();
efault:
        buf->error = -EFAULT;
        return false;
}

COMPAT_SYSCALL_DEFINE3(getdents, unsigned int, fd,
                struct compat_linux_dirent __user *, dirent, unsigned int, count)
{
        CLASS(fd_pos, f)(fd);
        struct compat_getdents_callback buf = {
                .ctx.actor = compat_filldir,
                .current_dir = dirent,
                .count = count
        };
        int error;

        if (fd_empty(f))
                return -EBADF;

        error = iterate_dir(fd_file(f), &buf.ctx);
        if (error >= 0)
                error = buf.error;
        if (buf.prev_reclen) {
                struct compat_linux_dirent __user * lastdirent;
                lastdirent = (void __user *)buf.current_dir - buf.prev_reclen;

                if (put_user(buf.ctx.pos, &lastdirent->d_off))
                        error = -EFAULT;
                else
                        error = count - buf.count;
        }
        return error;
}
#endif








































































































































































































































































































































































   69 











































































































































































































































 1215 






























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_KASAN_H
#define _LINUX_KASAN_H

#include <linux/bug.h>
#include <linux/kasan-enabled.h>
#include <linux/kasan-tags.h>
#include <linux/kernel.h>
#include <linux/static_key.h>
#include <linux/types.h>

struct kmem_cache;
struct page;
struct slab;
struct vm_struct;
struct task_struct;

#ifdef CONFIG_KASAN

#include <linux/linkage.h>
#include <asm/kasan.h>

#endif

typedef unsigned int __bitwise kasan_vmalloc_flags_t;

#define KASAN_VMALLOC_NONE                ((__force kasan_vmalloc_flags_t)0x00u)
#define KASAN_VMALLOC_INIT                ((__force kasan_vmalloc_flags_t)0x01u)
#define KASAN_VMALLOC_VM_ALLOC                ((__force kasan_vmalloc_flags_t)0x02u)
#define KASAN_VMALLOC_PROT_NORMAL        ((__force kasan_vmalloc_flags_t)0x04u)

#define KASAN_VMALLOC_PAGE_RANGE 0x1 /* Apply exsiting page range */
#define KASAN_VMALLOC_TLB_FLUSH  0x2 /* TLB flush */

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)

#include <linux/pgtable.h>

/* Software KASAN implementations use shadow memory. */

#ifdef CONFIG_KASAN_SW_TAGS
/* This matches KASAN_TAG_INVALID. */
#define KASAN_SHADOW_INIT 0xFE
#else
#define KASAN_SHADOW_INIT 0
#endif

#ifndef PTE_HWTABLE_PTRS
#define PTE_HWTABLE_PTRS 0
#endif

extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];
extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];
extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];
extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];

int kasan_populate_early_shadow(const void *shadow_start,
                                const void *shadow_end);

#ifndef kasan_mem_to_shadow
static inline void *kasan_mem_to_shadow(const void *addr)
{
        return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
                + KASAN_SHADOW_OFFSET;
}
#endif

int kasan_add_zero_shadow(void *start, unsigned long size);
void kasan_remove_zero_shadow(void *start, unsigned long size);

/* Enable reporting bugs after kasan_disable_current() */
extern void kasan_enable_current(void);

/* Disable reporting bugs for current task */
extern void kasan_disable_current(void);

#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

static inline int kasan_add_zero_shadow(void *start, unsigned long size)
{
        return 0;
}
static inline void kasan_remove_zero_shadow(void *start,
                                        unsigned long size)
{}

static inline void kasan_enable_current(void) {}
static inline void kasan_disable_current(void) {}

#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

#ifdef CONFIG_KASAN_HW_TAGS

#else /* CONFIG_KASAN_HW_TAGS */

#endif /* CONFIG_KASAN_HW_TAGS */

static inline bool kasan_has_integrated_init(void)
{
        return kasan_hw_tags_enabled();
}

#ifdef CONFIG_KASAN
void __kasan_unpoison_range(const void *addr, size_t size);
static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
{
        if (kasan_enabled())
                __kasan_unpoison_range(addr, size);
}

void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
static __always_inline void kasan_poison_pages(struct page *page,
                                                unsigned int order, bool init)
{
        if (kasan_enabled())
                __kasan_poison_pages(page, order, init);
}

bool __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
static __always_inline bool kasan_unpoison_pages(struct page *page,
                                                 unsigned int order, bool init)
{
        if (kasan_enabled())
                return __kasan_unpoison_pages(page, order, init);
        return false;
}

void __kasan_poison_slab(struct slab *slab);
static __always_inline void kasan_poison_slab(struct slab *slab)
{
        if (kasan_enabled())
                __kasan_poison_slab(slab);
}

void __kasan_unpoison_new_object(struct kmem_cache *cache, void *object);
/**
 * kasan_unpoison_new_object - Temporarily unpoison a new slab object.
 * @cache: Cache the object belong to.
 * @object: Pointer to the object.
 *
 * This function is intended for the slab allocator's internal use. It
 * temporarily unpoisons an object from a newly allocated slab without doing
 * anything else. The object must later be repoisoned by
 * kasan_poison_new_object().
 */
static __always_inline void kasan_unpoison_new_object(struct kmem_cache *cache,
                                                        void *object)
{
        if (kasan_enabled())
                __kasan_unpoison_new_object(cache, object);
}

void __kasan_poison_new_object(struct kmem_cache *cache, void *object);
/**
 * kasan_poison_new_object - Repoison a new slab object.
 * @cache: Cache the object belong to.
 * @object: Pointer to the object.
 *
 * This function is intended for the slab allocator's internal use. It
 * repoisons an object that was previously unpoisoned by
 * kasan_unpoison_new_object() without doing anything else.
 */
static __always_inline void kasan_poison_new_object(struct kmem_cache *cache,
                                                        void *object)
{
        if (kasan_enabled())
                __kasan_poison_new_object(cache, object);
}

void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
                                          const void *object);
static __always_inline void * __must_check kasan_init_slab_obj(
                                struct kmem_cache *cache, const void *object)
{
        if (kasan_enabled())
                return __kasan_init_slab_obj(cache, object);
        return (void *)object;
}

bool __kasan_slab_pre_free(struct kmem_cache *s, void *object,
                        unsigned long ip);
/**
 * kasan_slab_pre_free - Check whether freeing a slab object is safe.
 * @object: Object to be freed.
 *
 * This function checks whether freeing the given object is safe. It may
 * check for double-free and invalid-free bugs and report them.
 *
 * This function is intended only for use by the slab allocator.
 *
 * @Return true if freeing the object is unsafe; false otherwise.
 */
static __always_inline bool kasan_slab_pre_free(struct kmem_cache *s,
                                                void *object)
{
        if (kasan_enabled())
                return __kasan_slab_pre_free(s, object, _RET_IP_);
        return false;
}

bool __kasan_slab_free(struct kmem_cache *s, void *object, bool init,
                       bool still_accessible);
/**
 * kasan_slab_free - Poison, initialize, and quarantine a slab object.
 * @object: Object to be freed.
 * @init: Whether to initialize the object.
 * @still_accessible: Whether the object contents are still accessible.
 *
 * This function informs that a slab object has been freed and is not
 * supposed to be accessed anymore, except when @still_accessible is set
 * (indicating that the object is in a SLAB_TYPESAFE_BY_RCU cache and an RCU
 * grace period might not have passed yet).
 *
 * For KASAN modes that have integrated memory initialization
 * (kasan_has_integrated_init() == true), this function also initializes
 * the object's memory. For other modes, the @init argument is ignored.
 *
 * This function might also take ownership of the object to quarantine it.
 * When this happens, KASAN will defer freeing the object to a later
 * stage and handle it internally until then. The return value indicates
 * whether KASAN took ownership of the object.
 *
 * This function is intended only for use by the slab allocator.
 *
 * @Return true if KASAN took ownership of the object; false otherwise.
 */
static __always_inline bool kasan_slab_free(struct kmem_cache *s,
                                                void *object, bool init,
                                                bool still_accessible)
{
        if (kasan_enabled())
                return __kasan_slab_free(s, object, init, still_accessible);
        return false;
}

void __kasan_kfree_large(void *ptr, unsigned long ip);
static __always_inline void kasan_kfree_large(void *ptr)
{
        if (kasan_enabled())
                __kasan_kfree_large(ptr, _RET_IP_);
}

void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
                                       void *object, gfp_t flags, bool init);
static __always_inline void * __must_check kasan_slab_alloc(
                struct kmem_cache *s, void *object, gfp_t flags, bool init)
{
        if (kasan_enabled())
                return __kasan_slab_alloc(s, object, flags, init);
        return object;
}

void * __must_check __kasan_kmalloc(struct kmem_cache *s, const void *object,
                                    size_t size, gfp_t flags);
static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
                                const void *object, size_t size, gfp_t flags)
{
        if (kasan_enabled())
                return __kasan_kmalloc(s, object, size, flags);
        return (void *)object;
}

void * __must_check __kasan_kmalloc_large(const void *ptr,
                                          size_t size, gfp_t flags);
static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
                                                      size_t size, gfp_t flags)
{
        if (kasan_enabled())
                return __kasan_kmalloc_large(ptr, size, flags);
        return (void *)ptr;
}

void * __must_check __kasan_krealloc(const void *object,
                                     size_t new_size, gfp_t flags);
static __always_inline void * __must_check kasan_krealloc(const void *object,
                                                 size_t new_size, gfp_t flags)
{
        if (kasan_enabled())
                return __kasan_krealloc(object, new_size, flags);
        return (void *)object;
}

bool __kasan_mempool_poison_pages(struct page *page, unsigned int order,
                                  unsigned long ip);
/**
 * kasan_mempool_poison_pages - Check and poison a mempool page allocation.
 * @page: Pointer to the page allocation.
 * @order: Order of the allocation.
 *
 * This function is intended for kernel subsystems that cache page allocations
 * to reuse them instead of freeing them back to page_alloc (e.g. mempool).
 *
 * This function is similar to kasan_mempool_poison_object() but operates on
 * page allocations.
 *
 * Before the poisoned allocation can be reused, it must be unpoisoned via
 * kasan_mempool_unpoison_pages().
 *
 * Return: true if the allocation can be safely reused; false otherwise.
 */
static __always_inline bool kasan_mempool_poison_pages(struct page *page,
                                                       unsigned int order)
{
        if (kasan_enabled())
                return __kasan_mempool_poison_pages(page, order, _RET_IP_);
        return true;
}

void __kasan_mempool_unpoison_pages(struct page *page, unsigned int order,
                                    unsigned long ip);
/**
 * kasan_mempool_unpoison_pages - Unpoison a mempool page allocation.
 * @page: Pointer to the page allocation.
 * @order: Order of the allocation.
 *
 * This function is intended for kernel subsystems that cache page allocations
 * to reuse them instead of freeing them back to page_alloc (e.g. mempool).
 *
 * This function unpoisons a page allocation that was previously poisoned by
 * kasan_mempool_poison_pages() without zeroing the allocation's memory. For
 * the tag-based modes, this function assigns a new tag to the allocation.
 */
static __always_inline void kasan_mempool_unpoison_pages(struct page *page,
                                                         unsigned int order)
{
        if (kasan_enabled())
                __kasan_mempool_unpoison_pages(page, order, _RET_IP_);
}

bool __kasan_mempool_poison_object(void *ptr, unsigned long ip);
/**
 * kasan_mempool_poison_object - Check and poison a mempool slab allocation.
 * @ptr: Pointer to the slab allocation.
 *
 * This function is intended for kernel subsystems that cache slab allocations
 * to reuse them instead of freeing them back to the slab allocator (e.g.
 * mempool).
 *
 * This function poisons a slab allocation and saves a free stack trace for it
 * without initializing the allocation's memory and without putting it into the
 * quarantine (for the Generic mode).
 *
 * This function also performs checks to detect double-free and invalid-free
 * bugs and reports them. The caller can use the return value of this function
 * to find out if the allocation is buggy.
 *
 * Before the poisoned allocation can be reused, it must be unpoisoned via
 * kasan_mempool_unpoison_object().
 *
 * This function operates on all slab allocations including large kmalloc
 * allocations (the ones returned by kmalloc_large() or by kmalloc() with the
 * size > KMALLOC_MAX_SIZE).
 *
 * Return: true if the allocation can be safely reused; false otherwise.
 */
static __always_inline bool kasan_mempool_poison_object(void *ptr)
{
        if (kasan_enabled())
                return __kasan_mempool_poison_object(ptr, _RET_IP_);
        return true;
}

void __kasan_mempool_unpoison_object(void *ptr, size_t size, unsigned long ip);
/**
 * kasan_mempool_unpoison_object - Unpoison a mempool slab allocation.
 * @ptr: Pointer to the slab allocation.
 * @size: Size to be unpoisoned.
 *
 * This function is intended for kernel subsystems that cache slab allocations
 * to reuse them instead of freeing them back to the slab allocator (e.g.
 * mempool).
 *
 * This function unpoisons a slab allocation that was previously poisoned via
 * kasan_mempool_poison_object() and saves an alloc stack trace for it without
 * initializing the allocation's memory. For the tag-based modes, this function
 * does not assign a new tag to the allocation and instead restores the
 * original tags based on the pointer value.
 *
 * This function operates on all slab allocations including large kmalloc
 * allocations (the ones returned by kmalloc_large() or by kmalloc() with the
 * size > KMALLOC_MAX_SIZE).
 */
static __always_inline void kasan_mempool_unpoison_object(void *ptr,
                                                          size_t size)
{
        if (kasan_enabled())
                __kasan_mempool_unpoison_object(ptr, size, _RET_IP_);
}

/*
 * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
 * the hardware tag-based mode that doesn't rely on compiler instrumentation.
 */
bool __kasan_check_byte(const void *addr, unsigned long ip);
static __always_inline bool kasan_check_byte(const void *addr)
{
        if (kasan_enabled())
                return __kasan_check_byte(addr, _RET_IP_);
        return true;
}

#else /* CONFIG_KASAN */

static inline void kasan_unpoison_range(const void *address, size_t size) {}
static inline void kasan_poison_pages(struct page *page, unsigned int order,
                                      bool init) {}
static inline bool kasan_unpoison_pages(struct page *page, unsigned int order,
                                        bool init)
{
        return false;
}
static inline void kasan_poison_slab(struct slab *slab) {}
static inline void kasan_unpoison_new_object(struct kmem_cache *cache,
                                        void *object) {}
static inline void kasan_poison_new_object(struct kmem_cache *cache,
                                        void *object) {}
static inline void *kasan_init_slab_obj(struct kmem_cache *cache,
                                const void *object)
{
        return (void *)object;
}

static inline bool kasan_slab_pre_free(struct kmem_cache *s, void *object)
{
        return false;
}

static inline bool kasan_slab_free(struct kmem_cache *s, void *object,
                                   bool init, bool still_accessible)
{
        return false;
}
static inline void kasan_kfree_large(void *ptr) {}
static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object,
                                   gfp_t flags, bool init)
{
        return object;
}
static inline void *kasan_kmalloc(struct kmem_cache *s, const void *object,
                                size_t size, gfp_t flags)
{
        return (void *)object;
}
static inline void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
{
        return (void *)ptr;
}
static inline void *kasan_krealloc(const void *object, size_t new_size,
                                 gfp_t flags)
{
        return (void *)object;
}
static inline bool kasan_mempool_poison_pages(struct page *page, unsigned int order)
{
        return true;
}
static inline void kasan_mempool_unpoison_pages(struct page *page, unsigned int order) {}
static inline bool kasan_mempool_poison_object(void *ptr)
{
        return true;
}
static inline void kasan_mempool_unpoison_object(void *ptr, size_t size) {}

static inline bool kasan_check_byte(const void *address)
{
        return true;
}

#endif /* CONFIG_KASAN */

#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
void kasan_unpoison_task_stack(struct task_struct *task);
asmlinkage void kasan_unpoison_task_stack_below(const void *watermark);
#else
static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
static inline void kasan_unpoison_task_stack_below(const void *watermark) {}
#endif

#ifdef CONFIG_KASAN_GENERIC

struct kasan_cache {
        int alloc_meta_offset;
        int free_meta_offset;
};

size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object);
void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
                        slab_flags_t *flags);

void kasan_cache_shrink(struct kmem_cache *cache);
void kasan_cache_shutdown(struct kmem_cache *cache);
void kasan_record_aux_stack(void *ptr);

#else /* CONFIG_KASAN_GENERIC */

/* Tag-based KASAN modes do not use per-object metadata. */
static inline size_t kasan_metadata_size(struct kmem_cache *cache,
                                                bool in_object)
{
        return 0;
}
/* And no cache-related metadata initialization is required. */
static inline void kasan_cache_create(struct kmem_cache *cache,
                                      unsigned int *size,
                                      slab_flags_t *flags) {}

static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
static inline void kasan_record_aux_stack(void *ptr) {}

#endif /* CONFIG_KASAN_GENERIC */

#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)

static inline void *kasan_reset_tag(const void *addr)
{
        return (void *)arch_kasan_reset_tag(addr);
}

/**
 * kasan_report - print a report about a bad memory access detected by KASAN
 * @addr: address of the bad access
 * @size: size of the bad access
 * @is_write: whether the bad access is a write or a read
 * @ip: instruction pointer for the accessibility check or the bad access itself
 */
bool kasan_report(const void *addr, size_t size,
                bool is_write, unsigned long ip);

#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */

static inline void *kasan_reset_tag(const void *addr)
{
        return (void *)addr;
}

#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/

#ifdef CONFIG_KASAN_HW_TAGS

void kasan_report_async(void);

#endif /* CONFIG_KASAN_HW_TAGS */

#ifdef CONFIG_KASAN_SW_TAGS
void __init kasan_init_sw_tags(void);
#else
static inline void kasan_init_sw_tags(void) { }
#endif

#ifdef CONFIG_KASAN_HW_TAGS
void kasan_init_hw_tags_cpu(void);
void __init kasan_init_hw_tags(void);
#else
static inline void kasan_init_hw_tags_cpu(void) { }
static inline void kasan_init_hw_tags(void) { }
#endif

#ifdef CONFIG_KASAN_VMALLOC

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)

void kasan_populate_early_vm_area_shadow(void *start, unsigned long size);
int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
void kasan_release_vmalloc(unsigned long start, unsigned long end,
                           unsigned long free_region_start,
                           unsigned long free_region_end,
                           unsigned long flags);

#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

static inline void kasan_populate_early_vm_area_shadow(void *start,
                                                       unsigned long size)
{ }
static inline int kasan_populate_vmalloc(unsigned long start,
                                        unsigned long size)
{
        return 0;
}
static inline void kasan_release_vmalloc(unsigned long start,
                                         unsigned long end,
                                         unsigned long free_region_start,
                                         unsigned long free_region_end,
                                         unsigned long flags) { }

#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
                               kasan_vmalloc_flags_t flags);
static __always_inline void *kasan_unpoison_vmalloc(const void *start,
                                                unsigned long size,
                                                kasan_vmalloc_flags_t flags)
{
        if (kasan_enabled())
                return __kasan_unpoison_vmalloc(start, size, flags);
        return (void *)start;
}

void __kasan_poison_vmalloc(const void *start, unsigned long size);
static __always_inline void kasan_poison_vmalloc(const void *start,
                                                 unsigned long size)
{
        if (kasan_enabled())
                __kasan_poison_vmalloc(start, size);
}

#else /* CONFIG_KASAN_VMALLOC */

static inline void kasan_populate_early_vm_area_shadow(void *start,
                                                       unsigned long size) { }
static inline int kasan_populate_vmalloc(unsigned long start,
                                        unsigned long size)
{
        return 0;
}
static inline void kasan_release_vmalloc(unsigned long start,
                                         unsigned long end,
                                         unsigned long free_region_start,
                                         unsigned long free_region_end,
                                         unsigned long flags) { }

static inline void *kasan_unpoison_vmalloc(const void *start,
                                           unsigned long size,
                                           kasan_vmalloc_flags_t flags)
{
        return (void *)start;
}
static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
{ }

#endif /* CONFIG_KASAN_VMALLOC */

#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
                !defined(CONFIG_KASAN_VMALLOC)

/*
 * These functions allocate and free shadow memory for kernel modules.
 * They are only required when KASAN_VMALLOC is not supported, as otherwise
 * shadow memory is allocated by the generic vmalloc handlers.
 */
int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask);
void kasan_free_module_shadow(const struct vm_struct *vm);

#else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */

static inline int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) { return 0; }
static inline void kasan_free_module_shadow(const struct vm_struct *vm) {}

#endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
void kasan_non_canonical_hook(unsigned long addr);
#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
static inline void kasan_non_canonical_hook(unsigned long addr) { }
#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

#endif /* LINUX_KASAN_H */


















































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * fscrypt_private.h
 *
 * Copyright (C) 2015, Google, Inc.
 *
 * Originally written by Michael Halcrow, Ildar Muslukhov, and Uday Savagaonkar.
 * Heavily modified since then.
 */

#ifndef _FSCRYPT_PRIVATE_H
#define _FSCRYPT_PRIVATE_H

#include <linux/fscrypt.h>
#include <linux/siphash.h>
#include <crypto/hash.h>
#include <linux/blk-crypto.h>

#define CONST_STRLEN(str)        (sizeof(str) - 1)

#define FSCRYPT_FILE_NONCE_SIZE        16

/*
 * Minimum size of an fscrypt master key.  Note: a longer key will be required
 * if ciphers with a 256-bit security strength are used.  This is just the
 * absolute minimum, which applies when only 128-bit encryption is used.
 */
#define FSCRYPT_MIN_KEY_SIZE        16

#define FSCRYPT_CONTEXT_V1        1
#define FSCRYPT_CONTEXT_V2        2

/* Keep this in sync with include/uapi/linux/fscrypt.h */
#define FSCRYPT_MODE_MAX        FSCRYPT_MODE_AES_256_HCTR2

struct fscrypt_context_v1 {
        u8 version; /* FSCRYPT_CONTEXT_V1 */
        u8 contents_encryption_mode;
        u8 filenames_encryption_mode;
        u8 flags;
        u8 master_key_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE];
        u8 nonce[FSCRYPT_FILE_NONCE_SIZE];
};

struct fscrypt_context_v2 {
        u8 version; /* FSCRYPT_CONTEXT_V2 */
        u8 contents_encryption_mode;
        u8 filenames_encryption_mode;
        u8 flags;
        u8 log2_data_unit_size;
        u8 __reserved[3];
        u8 master_key_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE];
        u8 nonce[FSCRYPT_FILE_NONCE_SIZE];
};

/*
 * fscrypt_context - the encryption context of an inode
 *
 * This is the on-disk equivalent of an fscrypt_policy, stored alongside each
 * encrypted file usually in a hidden extended attribute.  It contains the
 * fields from the fscrypt_policy, in order to identify the encryption algorithm
 * and key with which the file is encrypted.  It also contains a nonce that was
 * randomly generated by fscrypt itself; this is used as KDF input or as a tweak
 * to cause different files to be encrypted differently.
 */
union fscrypt_context {
        u8 version;
        struct fscrypt_context_v1 v1;
        struct fscrypt_context_v2 v2;
};

/*
 * Return the size expected for the given fscrypt_context based on its version
 * number, or 0 if the context version is unrecognized.
 */
static inline int fscrypt_context_size(const union fscrypt_context *ctx)
{
        switch (ctx->version) {
        case FSCRYPT_CONTEXT_V1:
                BUILD_BUG_ON(sizeof(ctx->v1) != 28);
                return sizeof(ctx->v1);
        case FSCRYPT_CONTEXT_V2:
                BUILD_BUG_ON(sizeof(ctx->v2) != 40);
                return sizeof(ctx->v2);
        }
        return 0;
}

/* Check whether an fscrypt_context has a recognized version number and size */
static inline bool fscrypt_context_is_valid(const union fscrypt_context *ctx,
                                            int ctx_size)
{
        return ctx_size >= 1 && ctx_size == fscrypt_context_size(ctx);
}

/* Retrieve the context's nonce, assuming the context was already validated */
static inline const u8 *fscrypt_context_nonce(const union fscrypt_context *ctx)
{
        switch (ctx->version) {
        case FSCRYPT_CONTEXT_V1:
                return ctx->v1.nonce;
        case FSCRYPT_CONTEXT_V2:
                return ctx->v2.nonce;
        }
        WARN_ON_ONCE(1);
        return NULL;
}

union fscrypt_policy {
        u8 version;
        struct fscrypt_policy_v1 v1;
        struct fscrypt_policy_v2 v2;
};

/*
 * Return the size expected for the given fscrypt_policy based on its version
 * number, or 0 if the policy version is unrecognized.
 */
static inline int fscrypt_policy_size(const union fscrypt_policy *policy)
{
        switch (policy->version) {
        case FSCRYPT_POLICY_V1:
                return sizeof(policy->v1);
        case FSCRYPT_POLICY_V2:
                return sizeof(policy->v2);
        }
        return 0;
}

/* Return the contents encryption mode of a valid encryption policy */
static inline u8
fscrypt_policy_contents_mode(const union fscrypt_policy *policy)
{
        switch (policy->version) {
        case FSCRYPT_POLICY_V1:
                return policy->v1.contents_encryption_mode;
        case FSCRYPT_POLICY_V2:
                return policy->v2.contents_encryption_mode;
        }
        BUG();
}

/* Return the filenames encryption mode of a valid encryption policy */
static inline u8
fscrypt_policy_fnames_mode(const union fscrypt_policy *policy)
{
        switch (policy->version) {
        case FSCRYPT_POLICY_V1:
                return policy->v1.filenames_encryption_mode;
        case FSCRYPT_POLICY_V2:
                return policy->v2.filenames_encryption_mode;
        }
        BUG();
}

/* Return the flags (FSCRYPT_POLICY_FLAG*) of a valid encryption policy */
static inline u8
fscrypt_policy_flags(const union fscrypt_policy *policy)
{
        switch (policy->version) {
        case FSCRYPT_POLICY_V1:
                return policy->v1.flags;
        case FSCRYPT_POLICY_V2:
                return policy->v2.flags;
        }
        BUG();
}

static inline int
fscrypt_policy_v2_du_bits(const struct fscrypt_policy_v2 *policy,
                          const struct inode *inode)
{
        return policy->log2_data_unit_size ?: inode->i_blkbits;
}

static inline int
fscrypt_policy_du_bits(const union fscrypt_policy *policy,
                       const struct inode *inode)
{
        switch (policy->version) {
        case FSCRYPT_POLICY_V1:
                return inode->i_blkbits;
        case FSCRYPT_POLICY_V2:
                return fscrypt_policy_v2_du_bits(&policy->v2, inode);
        }
        BUG();
}

/*
 * For encrypted symlinks, the ciphertext length is stored at the beginning
 * of the string in little-endian format.
 */
struct fscrypt_symlink_data {
        __le16 len;
        char encrypted_path[];
} __packed;

/**
 * struct fscrypt_prepared_key - a key prepared for actual encryption/decryption
 * @tfm: crypto API transform object
 * @blk_key: key for blk-crypto
 *
 * Normally only one of the fields will be non-NULL.
 */
struct fscrypt_prepared_key {
        struct crypto_skcipher *tfm;
#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
        struct blk_crypto_key *blk_key;
#endif
};

/*
 * fscrypt_inode_info - the "encryption key" for an inode
 *
 * When an encrypted file's key is made available, an instance of this struct is
 * allocated and stored in ->i_crypt_info.  Once created, it remains until the
 * inode is evicted.
 */
struct fscrypt_inode_info {

        /* The key in a form prepared for actual encryption/decryption */
        struct fscrypt_prepared_key ci_enc_key;

        /* True if ci_enc_key should be freed when this struct is freed */
        u8 ci_owns_key : 1;

#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
        /*
         * True if this inode will use inline encryption (blk-crypto) instead of
         * the traditional filesystem-layer encryption.
         */
        u8 ci_inlinecrypt : 1;
#endif

        /* True if ci_dirhash_key is initialized */
        u8 ci_dirhash_key_initialized : 1;

        /*
         * log2 of the data unit size (granularity of contents encryption) of
         * this file.  This is computable from ci_policy and ci_inode but is
         * cached here for efficiency.  Only used for regular files.
         */
        u8 ci_data_unit_bits;

        /* Cached value: log2 of number of data units per FS block */
        u8 ci_data_units_per_block_bits;

        /* Hashed inode number.  Only set for IV_INO_LBLK_32 */
        u32 ci_hashed_ino;

        /*
         * Encryption mode used for this inode.  It corresponds to either the
         * contents or filenames encryption mode, depending on the inode type.
         */
        struct fscrypt_mode *ci_mode;

        /* Back-pointer to the inode */
        struct inode *ci_inode;

        /*
         * The master key with which this inode was unlocked (decrypted).  This
         * will be NULL if the master key was found in a process-subscribed
         * keyring rather than in the filesystem-level keyring.
         */
        struct fscrypt_master_key *ci_master_key;

        /*
         * Link in list of inodes that were unlocked with the master key.
         * Only used when ->ci_master_key is set.
         */
        struct list_head ci_master_key_link;

        /*
         * If non-NULL, then encryption is done using the master key directly
         * and ci_enc_key will equal ci_direct_key->dk_key.
         */
        struct fscrypt_direct_key *ci_direct_key;

        /*
         * This inode's hash key for filenames.  This is a 128-bit SipHash-2-4
         * key.  This is only set for directories that use a keyed dirhash over
         * the plaintext filenames -- currently just casefolded directories.
         */
        siphash_key_t ci_dirhash_key;

        /* The encryption policy used by this inode */
        union fscrypt_policy ci_policy;

        /* This inode's nonce, copied from the fscrypt_context */
        u8 ci_nonce[FSCRYPT_FILE_NONCE_SIZE];
};

typedef enum {
        FS_DECRYPT = 0,
        FS_ENCRYPT,
} fscrypt_direction_t;

/* crypto.c */
extern struct kmem_cache *fscrypt_inode_info_cachep;
int fscrypt_initialize(struct super_block *sb);
int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci,
                            fscrypt_direction_t rw, u64 index,
                            struct page *src_page, struct page *dest_page,
                            unsigned int len, unsigned int offs,
                            gfp_t gfp_flags);
struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags);

void __printf(3, 4) __cold
fscrypt_msg(const struct inode *inode, const char *level, const char *fmt, ...);

#define fscrypt_warn(inode, fmt, ...)                \
        fscrypt_msg((inode), KERN_WARNING, fmt, ##__VA_ARGS__)
#define fscrypt_err(inode, fmt, ...)                \
        fscrypt_msg((inode), KERN_ERR, fmt, ##__VA_ARGS__)

#define FSCRYPT_MAX_IV_SIZE        32

union fscrypt_iv {
        struct {
                /* zero-based index of data unit within the file */
                __le64 index;

                /* per-file nonce; only set in DIRECT_KEY mode */
                u8 nonce[FSCRYPT_FILE_NONCE_SIZE];
        };
        u8 raw[FSCRYPT_MAX_IV_SIZE];
        __le64 dun[FSCRYPT_MAX_IV_SIZE / sizeof(__le64)];
};

void fscrypt_generate_iv(union fscrypt_iv *iv, u64 index,
                         const struct fscrypt_inode_info *ci);

/*
 * Return the number of bits used by the maximum file data unit index that is
 * possible on the given filesystem, using the given log2 data unit size.
 */
static inline int
fscrypt_max_file_dun_bits(const struct super_block *sb, int du_bits)
{
        return fls64(sb->s_maxbytes - 1) - du_bits;
}

/* fname.c */
bool __fscrypt_fname_encrypted_size(const union fscrypt_policy *policy,
                                    u32 orig_len, u32 max_len,
                                    u32 *encrypted_len_ret);

/* hkdf.c */
struct fscrypt_hkdf {
        struct crypto_shash *hmac_tfm;
};

int fscrypt_init_hkdf(struct fscrypt_hkdf *hkdf, const u8 *master_key,
                      unsigned int master_key_size);

/*
 * The list of contexts in which fscrypt uses HKDF.  These values are used as
 * the first byte of the HKDF application-specific info string to guarantee that
 * info strings are never repeated between contexts.  This ensures that all HKDF
 * outputs are unique and cryptographically isolated, i.e. knowledge of one
 * output doesn't reveal another.
 */
#define HKDF_CONTEXT_KEY_IDENTIFIER        1 /* info=<empty>                */
#define HKDF_CONTEXT_PER_FILE_ENC_KEY        2 /* info=file_nonce                */
#define HKDF_CONTEXT_DIRECT_KEY                3 /* info=mode_num                */
#define HKDF_CONTEXT_IV_INO_LBLK_64_KEY        4 /* info=mode_num||fs_uuid        */
#define HKDF_CONTEXT_DIRHASH_KEY        5 /* info=file_nonce                */
#define HKDF_CONTEXT_IV_INO_LBLK_32_KEY        6 /* info=mode_num||fs_uuid        */
#define HKDF_CONTEXT_INODE_HASH_KEY        7 /* info=<empty>                */

int fscrypt_hkdf_expand(const struct fscrypt_hkdf *hkdf, u8 context,
                        const u8 *info, unsigned int infolen,
                        u8 *okm, unsigned int okmlen);

void fscrypt_destroy_hkdf(struct fscrypt_hkdf *hkdf);

/* inline_crypt.c */
#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
int fscrypt_select_encryption_impl(struct fscrypt_inode_info *ci);

static inline bool
fscrypt_using_inline_encryption(const struct fscrypt_inode_info *ci)
{
        return ci->ci_inlinecrypt;
}

int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
                                     const u8 *raw_key,
                                     const struct fscrypt_inode_info *ci);

void fscrypt_destroy_inline_crypt_key(struct super_block *sb,
                                      struct fscrypt_prepared_key *prep_key);

/*
 * Check whether the crypto transform or blk-crypto key has been allocated in
 * @prep_key, depending on which encryption implementation the file will use.
 */
static inline bool
fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
                        const struct fscrypt_inode_info *ci)
{
        /*
         * The two smp_load_acquire()'s here pair with the smp_store_release()'s
         * in fscrypt_prepare_inline_crypt_key() and fscrypt_prepare_key().
         * I.e., in some cases (namely, if this prep_key is a per-mode
         * encryption key) another task can publish blk_key or tfm concurrently,
         * executing a RELEASE barrier.  We need to use smp_load_acquire() here
         * to safely ACQUIRE the memory the other task published.
         */
        if (fscrypt_using_inline_encryption(ci))
                return smp_load_acquire(&prep_key->blk_key) != NULL;
        return smp_load_acquire(&prep_key->tfm) != NULL;
}

#else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */

static inline int fscrypt_select_encryption_impl(struct fscrypt_inode_info *ci)
{
        return 0;
}

static inline bool
fscrypt_using_inline_encryption(const struct fscrypt_inode_info *ci)
{
        return false;
}

static inline int
fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
                                 const u8 *raw_key,
                                 const struct fscrypt_inode_info *ci)
{
        WARN_ON_ONCE(1);
        return -EOPNOTSUPP;
}

static inline void
fscrypt_destroy_inline_crypt_key(struct super_block *sb,
                                 struct fscrypt_prepared_key *prep_key)
{
}

static inline bool
fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
                        const struct fscrypt_inode_info *ci)
{
        return smp_load_acquire(&prep_key->tfm) != NULL;
}
#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */

/* keyring.c */

/*
 * fscrypt_master_key_secret - secret key material of an in-use master key
 */
struct fscrypt_master_key_secret {

        /*
         * For v2 policy keys: HKDF context keyed by this master key.
         * For v1 policy keys: not set (hkdf.hmac_tfm == NULL).
         */
        struct fscrypt_hkdf        hkdf;

        /*
         * Size of the raw key in bytes.  This remains set even if ->raw was
         * zeroized due to no longer being needed.  I.e. we still remember the
         * size of the key even if we don't need to remember the key itself.
         */
        u32                        size;

        /* For v1 policy keys: the raw key.  Wiped for v2 policy keys. */
        u8                        raw[FSCRYPT_MAX_KEY_SIZE];

} __randomize_layout;

/*
 * fscrypt_master_key - an in-use master key
 *
 * This represents a master encryption key which has been added to the
 * filesystem.  There are three high-level states that a key can be in:
 *
 * FSCRYPT_KEY_STATUS_PRESENT
 *        Key is fully usable; it can be used to unlock inodes that are encrypted
 *        with it (this includes being able to create new inodes).  ->mk_present
 *        indicates whether the key is in this state.  ->mk_secret exists, the key
 *        is in the keyring, and ->mk_active_refs > 0 due to ->mk_present.
 *
 * FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED
 *        Removal of this key has been initiated, but some inodes that were
 *        unlocked with it are still in-use.  Like ABSENT, ->mk_secret is wiped,
 *        and the key can no longer be used to unlock inodes.  Unlike ABSENT, the
 *        key is still in the keyring; ->mk_decrypted_inodes is nonempty; and
 *        ->mk_active_refs > 0, being equal to the size of ->mk_decrypted_inodes.
 *
 *        This state transitions to ABSENT if ->mk_decrypted_inodes becomes empty,
 *        or to PRESENT if FS_IOC_ADD_ENCRYPTION_KEY is called again for this key.
 *
 * FSCRYPT_KEY_STATUS_ABSENT
 *        Key is fully removed.  The key is no longer in the keyring,
 *        ->mk_decrypted_inodes is empty, ->mk_active_refs == 0, ->mk_secret is
 *        wiped, and the key can no longer be used to unlock inodes.
 */
struct fscrypt_master_key {

        /*
         * Link in ->s_master_keys->key_hashtable.
         * Only valid if ->mk_active_refs > 0.
         */
        struct hlist_node                        mk_node;

        /* Semaphore that protects ->mk_secret, ->mk_users, and ->mk_present */
        struct rw_semaphore                        mk_sem;

        /*
         * Active and structural reference counts.  An active ref guarantees
         * that the struct continues to exist, continues to be in the keyring
         * ->s_master_keys, and that any embedded subkeys (e.g.
         * ->mk_direct_keys) that have been prepared continue to exist.
         * A structural ref only guarantees that the struct continues to exist.
         *
         * There is one active ref associated with ->mk_present being true, and
         * one active ref for each inode in ->mk_decrypted_inodes.
         *
         * There is one structural ref associated with the active refcount being
         * nonzero.  Finding a key in the keyring also takes a structural ref,
         * which is then held temporarily while the key is operated on.
         */
        refcount_t                                mk_active_refs;
        refcount_t                                mk_struct_refs;

        struct rcu_head                                mk_rcu_head;

        /*
         * The secret key material.  Wiped as soon as it is no longer needed;
         * for details, see the fscrypt_master_key struct comment.
         *
         * Locking: protected by ->mk_sem.
         */
        struct fscrypt_master_key_secret        mk_secret;

        /*
         * For v1 policy keys: an arbitrary key descriptor which was assigned by
         * userspace (->descriptor).
         *
         * For v2 policy keys: a cryptographic hash of this key (->identifier).
         */
        struct fscrypt_key_specifier                mk_spec;

        /*
         * Keyring which contains a key of type 'key_type_fscrypt_user' for each
         * user who has added this key.  Normally each key will be added by just
         * one user, but it's possible that multiple users share a key, and in
         * that case we need to keep track of those users so that one user can't
         * remove the key before the others want it removed too.
         *
         * This is NULL for v1 policy keys; those can only be added by root.
         *
         * Locking: protected by ->mk_sem.  (We don't just rely on the keyrings
         * subsystem semaphore ->mk_users->sem, as we need support for atomic
         * search+insert along with proper synchronization with other fields.)
         */
        struct key                *mk_users;

        /*
         * List of inodes that were unlocked using this key.  This allows the
         * inodes to be evicted efficiently if the key is removed.
         */
        struct list_head        mk_decrypted_inodes;
        spinlock_t                mk_decrypted_inodes_lock;

        /*
         * Per-mode encryption keys for the various types of encryption policies
         * that use them.  Allocated and derived on-demand.
         */
        struct fscrypt_prepared_key mk_direct_keys[FSCRYPT_MODE_MAX + 1];
        struct fscrypt_prepared_key mk_iv_ino_lblk_64_keys[FSCRYPT_MODE_MAX + 1];
        struct fscrypt_prepared_key mk_iv_ino_lblk_32_keys[FSCRYPT_MODE_MAX + 1];

        /* Hash key for inode numbers.  Initialized only when needed. */
        siphash_key_t                mk_ino_hash_key;
        bool                        mk_ino_hash_key_initialized;

        /*
         * Whether this key is in the "present" state, i.e. fully usable.  For
         * details, see the fscrypt_master_key struct comment.
         *
         * Locking: protected by ->mk_sem, but can be read locklessly using
         * READ_ONCE().  Writers must use WRITE_ONCE() when concurrent readers
         * are possible.
         */
        bool                        mk_present;

} __randomize_layout;

static inline const char *master_key_spec_type(
                                const struct fscrypt_key_specifier *spec)
{
        switch (spec->type) {
        case FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR:
                return "descriptor";
        case FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER:
                return "identifier";
        }
        return "[unknown]";
}

static inline int master_key_spec_len(const struct fscrypt_key_specifier *spec)
{
        switch (spec->type) {
        case FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR:
                return FSCRYPT_KEY_DESCRIPTOR_SIZE;
        case FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER:
                return FSCRYPT_KEY_IDENTIFIER_SIZE;
        }
        return 0;
}

void fscrypt_put_master_key(struct fscrypt_master_key *mk);

void fscrypt_put_master_key_activeref(struct super_block *sb,
                                      struct fscrypt_master_key *mk);

struct fscrypt_master_key *
fscrypt_find_master_key(struct super_block *sb,
                        const struct fscrypt_key_specifier *mk_spec);

int fscrypt_get_test_dummy_key_identifier(
                          u8 key_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]);

int fscrypt_add_test_dummy_key(struct super_block *sb,
                               struct fscrypt_key_specifier *key_spec);

int fscrypt_verify_key_added(struct super_block *sb,
                             const u8 identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]);

int __init fscrypt_init_keyring(void);

/* keysetup.c */

struct fscrypt_mode {
        const char *friendly_name;
        const char *cipher_str;
        int keysize;                /* key size in bytes */
        int security_strength;        /* security strength in bytes */
        int ivsize;                /* IV size in bytes */
        int logged_cryptoapi_impl;
        int logged_blk_crypto_native;
        int logged_blk_crypto_fallback;
        enum blk_crypto_mode_num blk_crypto_mode;
};

extern struct fscrypt_mode fscrypt_modes[];

int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key,
                        const u8 *raw_key, const struct fscrypt_inode_info *ci);

void fscrypt_destroy_prepared_key(struct super_block *sb,
                                  struct fscrypt_prepared_key *prep_key);

int fscrypt_set_per_file_enc_key(struct fscrypt_inode_info *ci,
                                 const u8 *raw_key);

int fscrypt_derive_dirhash_key(struct fscrypt_inode_info *ci,
                               const struct fscrypt_master_key *mk);

void fscrypt_hash_inode_number(struct fscrypt_inode_info *ci,
                               const struct fscrypt_master_key *mk);

int fscrypt_get_encryption_info(struct inode *inode, bool allow_unsupported);

/**
 * fscrypt_require_key() - require an inode's encryption key
 * @inode: the inode we need the key for
 *
 * If the inode is encrypted, set up its encryption key if not already done.
 * Then require that the key be present and return -ENOKEY otherwise.
 *
 * No locks are needed, and the key will live as long as the struct inode --- so
 * it won't go away from under you.
 *
 * Return: 0 on success, -ENOKEY if the key is missing, or another -errno code
 * if a problem occurred while setting up the encryption key.
 */
static inline int fscrypt_require_key(struct inode *inode)
{
        if (IS_ENCRYPTED(inode)) {
                int err = fscrypt_get_encryption_info(inode, false);

                if (err)
                        return err;
                if (!fscrypt_has_encryption_key(inode))
                        return -ENOKEY;
        }
        return 0;
}

/* keysetup_v1.c */

void fscrypt_put_direct_key(struct fscrypt_direct_key *dk);

int fscrypt_setup_v1_file_key(struct fscrypt_inode_info *ci,
                              const u8 *raw_master_key);

int fscrypt_setup_v1_file_key_via_subscribed_keyrings(
                                struct fscrypt_inode_info *ci);

/* policy.c */

bool fscrypt_policies_equal(const union fscrypt_policy *policy1,
                            const union fscrypt_policy *policy2);
int fscrypt_policy_to_key_spec(const union fscrypt_policy *policy,
                               struct fscrypt_key_specifier *key_spec);
const union fscrypt_policy *fscrypt_get_dummy_policy(struct super_block *sb);
bool fscrypt_supported_policy(const union fscrypt_policy *policy_u,
                              const struct inode *inode);
int fscrypt_policy_from_context(union fscrypt_policy *policy_u,
                                const union fscrypt_context *ctx_u,
                                int ctx_size);
const union fscrypt_policy *fscrypt_policy_to_inherit(struct inode *dir);

#endif /* _FSCRYPT_PRIVATE_H */



















































    1 













    1 




    3 

    1 



























































































































































































































































































































































































































    1 





    1 








    1 















    1 








    3 





    3 


    3 

    3 








    2 

    1 

    1 
























    2 









    3 






    1 

    1 


    1 



































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2005-2008 Red Hat, Inc.  All rights reserved.
 */

#include <linux/fs.h>
#include <linux/filelock.h>
#include <linux/miscdevice.h>
#include <linux/poll.h>
#include <linux/dlm.h>
#include <linux/dlm_plock.h>
#include <linux/slab.h>

#include <trace/events/dlm.h>

#include "dlm_internal.h"
#include "lockspace.h"

static DEFINE_SPINLOCK(ops_lock);
static LIST_HEAD(send_list);
static LIST_HEAD(recv_list);
static DECLARE_WAIT_QUEUE_HEAD(send_wq);
static DECLARE_WAIT_QUEUE_HEAD(recv_wq);

struct plock_async_data {
        void *fl;
        void *file;
        struct file_lock flc;
        int (*callback)(struct file_lock *fl, int result);
};

struct plock_op {
        struct list_head list;
        int done;
        struct dlm_plock_info info;
        /* if set indicates async handling */
        struct plock_async_data *data;
};

static inline void set_version(struct dlm_plock_info *info)
{
        info->version[0] = DLM_PLOCK_VERSION_MAJOR;
        info->version[1] = DLM_PLOCK_VERSION_MINOR;
        info->version[2] = DLM_PLOCK_VERSION_PATCH;
}

static struct plock_op *plock_lookup_waiter(const struct dlm_plock_info *info)
{
        struct plock_op *op = NULL, *iter;

        list_for_each_entry(iter, &recv_list, list) {
                if (iter->info.fsid == info->fsid &&
                    iter->info.number == info->number &&
                    iter->info.owner == info->owner &&
                    iter->info.pid == info->pid &&
                    iter->info.start == info->start &&
                    iter->info.end == info->end &&
                    iter->info.ex == info->ex &&
                    iter->info.wait) {
                        op = iter;
                        break;
                }
        }

        return op;
}

static int check_version(struct dlm_plock_info *info)
{
        if ((DLM_PLOCK_VERSION_MAJOR != info->version[0]) ||
            (DLM_PLOCK_VERSION_MINOR < info->version[1])) {
                log_print("plock device version mismatch: "
                          "kernel (%u.%u.%u), user (%u.%u.%u)",
                          DLM_PLOCK_VERSION_MAJOR,
                          DLM_PLOCK_VERSION_MINOR,
                          DLM_PLOCK_VERSION_PATCH,
                          info->version[0],
                          info->version[1],
                          info->version[2]);
                return -EINVAL;
        }
        return 0;
}

static void dlm_release_plock_op(struct plock_op *op)
{
        kfree(op->data);
        kfree(op);
}

static void send_op(struct plock_op *op)
{
        set_version(&op->info);
        spin_lock(&ops_lock);
        list_add_tail(&op->list, &send_list);
        spin_unlock(&ops_lock);
        wake_up(&send_wq);
}

static int do_lock_cancel(const struct dlm_plock_info *orig_info)
{
        struct plock_op *op;
        int rv;

        op = kzalloc(sizeof(*op), GFP_NOFS);
        if (!op)
                return -ENOMEM;

        op->info = *orig_info;
        op->info.optype = DLM_PLOCK_OP_CANCEL;
        op->info.wait = 0;

        send_op(op);
        wait_event(recv_wq, (op->done != 0));

        rv = op->info.rv;

        dlm_release_plock_op(op);
        return rv;
}

int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file,
                   int cmd, struct file_lock *fl)
{
        struct plock_async_data *op_data;
        struct dlm_ls *ls;
        struct plock_op *op;
        int rv;

        ls = dlm_find_lockspace_local(lockspace);
        if (!ls)
                return -EINVAL;

        op = kzalloc(sizeof(*op), GFP_NOFS);
        if (!op) {
                rv = -ENOMEM;
                goto out;
        }

        op->info.optype                = DLM_PLOCK_OP_LOCK;
        op->info.pid                = fl->c.flc_pid;
        op->info.ex                = lock_is_write(fl);
        op->info.wait                = !!(fl->c.flc_flags & FL_SLEEP);
        op->info.fsid                = ls->ls_global_id;
        op->info.number                = number;
        op->info.start                = fl->fl_start;
        op->info.end                = fl->fl_end;
        op->info.owner = (__u64)(long) fl->c.flc_owner;
        /* async handling */
        if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
                op_data = kzalloc(sizeof(*op_data), GFP_NOFS);
                if (!op_data) {
                        dlm_release_plock_op(op);
                        rv = -ENOMEM;
                        goto out;
                }

                op_data->callback = fl->fl_lmops->lm_grant;
                locks_init_lock(&op_data->flc);
                locks_copy_lock(&op_data->flc, fl);
                op_data->fl                = fl;
                op_data->file        = file;

                op->data = op_data;

                send_op(op);
                rv = FILE_LOCK_DEFERRED;
                goto out;
        }

        send_op(op);

        if (op->info.wait) {
                rv = wait_event_interruptible(recv_wq, (op->done != 0));
                if (rv == -ERESTARTSYS) {
                        spin_lock(&ops_lock);
                        /* recheck under ops_lock if we got a done != 0,
                         * if so this interrupt case should be ignored
                         */
                        if (op->done != 0) {
                                spin_unlock(&ops_lock);
                                goto do_lock_wait;
                        }
                        spin_unlock(&ops_lock);

                        rv = do_lock_cancel(&op->info);
                        switch (rv) {
                        case 0:
                                /* waiter was deleted in user space, answer will never come
                                 * remove original request. The original request must be
                                 * on recv_list because the answer of do_lock_cancel()
                                 * synchronized it.
                                 */
                                spin_lock(&ops_lock);
                                list_del(&op->list);
                                spin_unlock(&ops_lock);
                                rv = -EINTR;
                                break;
                        case -ENOENT:
                                /* cancellation wasn't successful but op should be done */
                                fallthrough;
                        default:
                                /* internal error doing cancel we need to wait */
                                goto wait;
                        }

                        log_debug(ls, "%s: wait interrupted %x %llx pid %d",
                                  __func__, ls->ls_global_id,
                                  (unsigned long long)number, op->info.pid);
                        dlm_release_plock_op(op);
                        goto out;
                }
        } else {
wait:
                wait_event(recv_wq, (op->done != 0));
        }

do_lock_wait:

        WARN_ON(!list_empty(&op->list));

        rv = op->info.rv;

        if (!rv) {
                if (locks_lock_file_wait(file, fl) < 0)
                        log_error(ls, "dlm_posix_lock: vfs lock error %llx",
                                  (unsigned long long)number);
        }

        dlm_release_plock_op(op);
out:
        dlm_put_lockspace(ls);
        return rv;
}
EXPORT_SYMBOL_GPL(dlm_posix_lock);

/* Returns failure iff a successful lock operation should be canceled */
static int dlm_plock_callback(struct plock_op *op)
{
        struct plock_async_data *op_data = op->data;
        struct file *file;
        struct file_lock *fl;
        struct file_lock *flc;
        int (*notify)(struct file_lock *fl, int result) = NULL;
        int rv = 0;

        WARN_ON(!list_empty(&op->list));

        /* check if the following 2 are still valid or make a copy */
        file = op_data->file;
        flc = &op_data->flc;
        fl = op_data->fl;
        notify = op_data->callback;

        if (op->info.rv) {
                notify(fl, op->info.rv);
                goto out;
        }

        /* got fs lock; bookkeep locally as well: */
        flc->c.flc_flags &= ~FL_SLEEP;
        if (posix_lock_file(file, flc, NULL)) {
                /*
                 * This can only happen in the case of kmalloc() failure.
                 * The filesystem's own lock is the authoritative lock,
                 * so a failure to get the lock locally is not a disaster.
                 * As long as the fs cannot reliably cancel locks (especially
                 * in a low-memory situation), we're better off ignoring
                 * this failure than trying to recover.
                 */
                log_print("dlm_plock_callback: vfs lock error %llx file %p fl %p",
                          (unsigned long long)op->info.number, file, fl);
        }

        rv = notify(fl, 0);
        if (rv) {
                /* XXX: We need to cancel the fs lock here: */
                log_print("%s: lock granted after lock request failed; dangling lock!",
                          __func__);
                goto out;
        }

out:
        dlm_release_plock_op(op);
        return rv;
}

int dlm_posix_unlock(dlm_lockspace_t *lockspace, u64 number, struct file *file,
                     struct file_lock *fl)
{
        struct dlm_ls *ls;
        struct plock_op *op;
        int rv;
        unsigned char saved_flags = fl->c.flc_flags;

        ls = dlm_find_lockspace_local(lockspace);
        if (!ls)
                return -EINVAL;

        op = kzalloc(sizeof(*op), GFP_NOFS);
        if (!op) {
                rv = -ENOMEM;
                goto out;
        }

        /* cause the vfs unlock to return ENOENT if lock is not found */
        fl->c.flc_flags |= FL_EXISTS;

        rv = locks_lock_file_wait(file, fl);
        if (rv == -ENOENT) {
                rv = 0;
                goto out_free;
        }
        if (rv < 0) {
                log_error(ls, "dlm_posix_unlock: vfs unlock error %d %llx",
                          rv, (unsigned long long)number);
        }

        op->info.optype                = DLM_PLOCK_OP_UNLOCK;
        op->info.pid                = fl->c.flc_pid;
        op->info.fsid                = ls->ls_global_id;
        op->info.number                = number;
        op->info.start                = fl->fl_start;
        op->info.end                = fl->fl_end;
        op->info.owner = (__u64)(long) fl->c.flc_owner;

        if (fl->c.flc_flags & FL_CLOSE) {
                op->info.flags |= DLM_PLOCK_FL_CLOSE;
                send_op(op);
                rv = 0;
                goto out;
        }

        send_op(op);
        wait_event(recv_wq, (op->done != 0));

        WARN_ON(!list_empty(&op->list));

        rv = op->info.rv;

        if (rv == -ENOENT)
                rv = 0;

out_free:
        dlm_release_plock_op(op);
out:
        dlm_put_lockspace(ls);
        fl->c.flc_flags = saved_flags;
        return rv;
}
EXPORT_SYMBOL_GPL(dlm_posix_unlock);

/*
 * NOTE: This implementation can only handle async lock requests as nfs
 * do it. It cannot handle cancellation of a pending lock request sitting
 * in wait_event(), but for now only nfs is the only user local kernel
 * user.
 */
int dlm_posix_cancel(dlm_lockspace_t *lockspace, u64 number, struct file *file,
                     struct file_lock *fl)
{
        struct dlm_plock_info info;
        struct plock_op *op;
        struct dlm_ls *ls;
        int rv;

        /* this only works for async request for now and nfs is the only
         * kernel user right now.
         */
        if (WARN_ON_ONCE(!fl->fl_lmops || !fl->fl_lmops->lm_grant))
                return -EOPNOTSUPP;

        ls = dlm_find_lockspace_local(lockspace);
        if (!ls)
                return -EINVAL;

        memset(&info, 0, sizeof(info));
        info.pid = fl->c.flc_pid;
        info.ex = lock_is_write(fl);
        info.fsid = ls->ls_global_id;
        dlm_put_lockspace(ls);
        info.number = number;
        info.start = fl->fl_start;
        info.end = fl->fl_end;
        info.owner = (__u64)(long) fl->c.flc_owner;

        rv = do_lock_cancel(&info);
        switch (rv) {
        case 0:
                spin_lock(&ops_lock);
                /* lock request to cancel must be on recv_list because
                 * do_lock_cancel() synchronizes it.
                 */
                op = plock_lookup_waiter(&info);
                if (WARN_ON_ONCE(!op)) {
                        spin_unlock(&ops_lock);
                        rv = -ENOLCK;
                        break;
                }

                list_del(&op->list);
                spin_unlock(&ops_lock);
                WARN_ON(op->info.optype != DLM_PLOCK_OP_LOCK);
                op->data->callback(op->data->fl, -EINTR);
                dlm_release_plock_op(op);
                rv = -EINTR;
                break;
        case -ENOENT:
                /* if cancel wasn't successful we probably were to late
                 * or it was a non-blocking lock request, so just unlock it.
                 */
                rv = dlm_posix_unlock(lockspace, number, file, fl);
                break;
        default:
                break;
        }

        return rv;
}
EXPORT_SYMBOL_GPL(dlm_posix_cancel);

int dlm_posix_get(dlm_lockspace_t *lockspace, u64 number, struct file *file,
                  struct file_lock *fl)
{
        struct dlm_ls *ls;
        struct plock_op *op;
        int rv;

        ls = dlm_find_lockspace_local(lockspace);
        if (!ls)
                return -EINVAL;

        op = kzalloc(sizeof(*op), GFP_NOFS);
        if (!op) {
                rv = -ENOMEM;
                goto out;
        }

        op->info.optype                = DLM_PLOCK_OP_GET;
        op->info.pid                = fl->c.flc_pid;
        op->info.ex                = lock_is_write(fl);
        op->info.fsid                = ls->ls_global_id;
        op->info.number                = number;
        op->info.start                = fl->fl_start;
        op->info.end                = fl->fl_end;
        op->info.owner = (__u64)(long) fl->c.flc_owner;

        send_op(op);
        wait_event(recv_wq, (op->done != 0));

        WARN_ON(!list_empty(&op->list));

        /* info.rv from userspace is 1 for conflict, 0 for no-conflict,
           -ENOENT if there are no locks on the file */

        rv = op->info.rv;

        fl->c.flc_type = F_UNLCK;
        if (rv == -ENOENT)
                rv = 0;
        else if (rv > 0) {
                locks_init_lock(fl);
                fl->c.flc_type = (op->info.ex) ? F_WRLCK : F_RDLCK;
                fl->c.flc_flags = FL_POSIX;
                fl->c.flc_pid = op->info.pid;
                if (op->info.nodeid != dlm_our_nodeid())
                        fl->c.flc_pid = -fl->c.flc_pid;
                fl->fl_start = op->info.start;
                fl->fl_end = op->info.end;
                rv = 0;
        }

        dlm_release_plock_op(op);
out:
        dlm_put_lockspace(ls);
        return rv;
}
EXPORT_SYMBOL_GPL(dlm_posix_get);

/* a read copies out one plock request from the send list */
static ssize_t dev_read(struct file *file, char __user *u, size_t count,
                        loff_t *ppos)
{
        struct dlm_plock_info info;
        struct plock_op *op = NULL;

        if (count < sizeof(info))
                return -EINVAL;

        spin_lock(&ops_lock);
        if (!list_empty(&send_list)) {
                op = list_first_entry(&send_list, struct plock_op, list);
                if (op->info.flags & DLM_PLOCK_FL_CLOSE)
                        list_del(&op->list);
                else
                        list_move_tail(&op->list, &recv_list);
                memcpy(&info, &op->info, sizeof(info));
        }
        spin_unlock(&ops_lock);

        if (!op)
                return -EAGAIN;

        trace_dlm_plock_read(&info);

        /* there is no need to get a reply from userspace for unlocks
           that were generated by the vfs cleaning up for a close
           (the process did not make an unlock call). */

        if (op->info.flags & DLM_PLOCK_FL_CLOSE)
                dlm_release_plock_op(op);

        if (copy_to_user(u, &info, sizeof(info)))
                return -EFAULT;
        return sizeof(info);
}

/* a write copies in one plock result that should match a plock_op
   on the recv list */
static ssize_t dev_write(struct file *file, const char __user *u, size_t count,
                         loff_t *ppos)
{
        struct plock_op *op = NULL, *iter;
        struct dlm_plock_info info;
        int do_callback = 0;

        if (count != sizeof(info))
                return -EINVAL;

        if (copy_from_user(&info, u, sizeof(info)))
                return -EFAULT;

        trace_dlm_plock_write(&info);

        if (check_version(&info))
                return -EINVAL;

        /*
         * The results for waiting ops (SETLKW) can be returned in any
         * order, so match all fields to find the op.  The results for
         * non-waiting ops are returned in the order that they were sent
         * to userspace, so match the result with the first non-waiting op.
         */
        spin_lock(&ops_lock);
        if (info.wait) {
                op = plock_lookup_waiter(&info);
        } else {
                list_for_each_entry(iter, &recv_list, list) {
                        if (!iter->info.wait &&
                            iter->info.fsid == info.fsid) {
                                op = iter;
                                break;
                        }
                }
        }

        if (op) {
                /* Sanity check that op and info match. */
                if (info.wait)
                        WARN_ON(op->info.optype != DLM_PLOCK_OP_LOCK);
                else
                        WARN_ON(op->info.number != info.number ||
                                op->info.owner != info.owner ||
                                op->info.optype != info.optype);

                list_del_init(&op->list);
                memcpy(&op->info, &info, sizeof(info));
                if (op->data)
                        do_callback = 1;
                else
                        op->done = 1;
        }
        spin_unlock(&ops_lock);

        if (op) {
                if (do_callback)
                        dlm_plock_callback(op);
                else
                        wake_up(&recv_wq);
        } else
                pr_debug("%s: no op %x %llx", __func__,
                         info.fsid, (unsigned long long)info.number);
        return count;
}

static __poll_t dev_poll(struct file *file, poll_table *wait)
{
        __poll_t mask = 0;

        poll_wait(file, &send_wq, wait);

        spin_lock(&ops_lock);
        if (!list_empty(&send_list))
                mask = EPOLLIN | EPOLLRDNORM;
        spin_unlock(&ops_lock);

        return mask;
}

static const struct file_operations dev_fops = {
        .read    = dev_read,
        .write   = dev_write,
        .poll    = dev_poll,
        .owner   = THIS_MODULE,
        .llseek  = noop_llseek,
};

static struct miscdevice plock_dev_misc = {
        .minor = MISC_DYNAMIC_MINOR,
        .name = DLM_PLOCK_MISC_NAME,
        .fops = &dev_fops
};

int dlm_plock_init(void)
{
        int rv;

        rv = misc_register(&plock_dev_misc);
        if (rv)
                log_print("dlm_plock_init: misc_register failed %d", rv);
        return rv;
}

void dlm_plock_exit(void)
{
        misc_deregister(&plock_dev_misc);
        WARN_ON(!list_empty(&send_list));
        WARN_ON(!list_empty(&recv_list));
}






















    1 









    1 

    1 




    1 














    2 




    1 




    1 

    2 




































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Cryptographic API.
 *
 * DES & Triple DES EDE Cipher Algorithms.
 *
 * Copyright (c) 2005 Dag Arne Osvik <da@osvik.no>
 */

#include <asm/byteorder.h>
#include <crypto/algapi.h>
#include <linux/bitops.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/errno.h>

#include <crypto/internal/des.h>

static int des_setkey(struct crypto_tfm *tfm, const u8 *key,
                      unsigned int keylen)
{
        struct des_ctx *dctx = crypto_tfm_ctx(tfm);
        int err;

        err = des_expand_key(dctx, key, keylen);
        if (err == -ENOKEY) {
                if (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)
                        err = -EINVAL;
                else
                        err = 0;
        }
        if (err)
                memset(dctx, 0, sizeof(*dctx));
        return err;
}

static void crypto_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        const struct des_ctx *dctx = crypto_tfm_ctx(tfm);

        des_encrypt(dctx, dst, src);
}

static void crypto_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        const struct des_ctx *dctx = crypto_tfm_ctx(tfm);

        des_decrypt(dctx, dst, src);
}

static int des3_ede_setkey(struct crypto_tfm *tfm, const u8 *key,
                           unsigned int keylen)
{
        struct des3_ede_ctx *dctx = crypto_tfm_ctx(tfm);
        int err;

        err = des3_ede_expand_key(dctx, key, keylen);
        if (err == -ENOKEY) {
                if (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)
                        err = -EINVAL;
                else
                        err = 0;
        }
        if (err)
                memset(dctx, 0, sizeof(*dctx));
        return err;
}

static void crypto_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst,
                                    const u8 *src)
{
        const struct des3_ede_ctx *dctx = crypto_tfm_ctx(tfm);

        des3_ede_encrypt(dctx, dst, src);
}

static void crypto_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst,
                                    const u8 *src)
{
        const struct des3_ede_ctx *dctx = crypto_tfm_ctx(tfm);

        des3_ede_decrypt(dctx, dst, src);
}

static struct crypto_alg des_algs[2] = { {
        .cra_name                =        "des",
        .cra_driver_name        =        "des-generic",
        .cra_priority                =        100,
        .cra_flags                =        CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                =        DES_BLOCK_SIZE,
        .cra_ctxsize                =        sizeof(struct des_ctx),
        .cra_module                =        THIS_MODULE,
        .cra_u                        =        { .cipher = {
        .cia_min_keysize        =        DES_KEY_SIZE,
        .cia_max_keysize        =        DES_KEY_SIZE,
        .cia_setkey                =        des_setkey,
        .cia_encrypt                =        crypto_des_encrypt,
        .cia_decrypt                =        crypto_des_decrypt } }
}, {
        .cra_name                =        "des3_ede",
        .cra_driver_name        =        "des3_ede-generic",
        .cra_priority                =        100,
        .cra_flags                =        CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                =        DES3_EDE_BLOCK_SIZE,
        .cra_ctxsize                =        sizeof(struct des3_ede_ctx),
        .cra_module                =        THIS_MODULE,
        .cra_u                        =        { .cipher = {
        .cia_min_keysize        =        DES3_EDE_KEY_SIZE,
        .cia_max_keysize        =        DES3_EDE_KEY_SIZE,
        .cia_setkey                =        des3_ede_setkey,
        .cia_encrypt                =        crypto_des3_ede_encrypt,
        .cia_decrypt                =        crypto_des3_ede_decrypt } }
} };

static int __init des_generic_mod_init(void)
{
        return crypto_register_algs(des_algs, ARRAY_SIZE(des_algs));
}

static void __exit des_generic_mod_fini(void)
{
        crypto_unregister_algs(des_algs, ARRAY_SIZE(des_algs));
}

subsys_initcall(des_generic_mod_init);
module_exit(des_generic_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms");
MODULE_AUTHOR("Dag Arne Osvik <da@osvik.no>");
MODULE_ALIAS_CRYPTO("des");
MODULE_ALIAS_CRYPTO("des-generic");
MODULE_ALIAS_CRYPTO("des3_ede");
MODULE_ALIAS_CRYPTO("des3_ede-generic");




























































































    1 

























    1 




























    1 







































































    1 
    1 

















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * devres.c - managed gpio resources
 * This file is based on kernel/irq/devres.c
 *
 * Copyright (c) 2011 John Crispin <john@phrozen.org>
 */

#include <linux/device.h>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/gfp.h>
#include <linux/types.h>

#include <linux/gpio/consumer.h>

#include "gpiolib.h"

struct fwnode_handle;
struct lock_class_key;

static void devm_gpiod_release(struct device *dev, void *res)
{
        struct gpio_desc **desc = res;

        gpiod_put(*desc);
}

static int devm_gpiod_match(struct device *dev, void *res, void *data)
{
        struct gpio_desc **this = res, **gpio = data;

        return *this == *gpio;
}

static void devm_gpiod_release_array(struct device *dev, void *res)
{
        struct gpio_descs **descs = res;

        gpiod_put_array(*descs);
}

static int devm_gpiod_match_array(struct device *dev, void *res, void *data)
{
        struct gpio_descs **this = res, **gpios = data;

        return *this == *gpios;
}

/**
 * devm_gpiod_get - Resource-managed gpiod_get()
 * @dev:        GPIO consumer
 * @con_id:        function within the GPIO consumer
 * @flags:        optional GPIO initialization flags
 *
 * Managed gpiod_get(). GPIO descriptors returned from this function are
 * automatically disposed on driver detach. See gpiod_get() for detailed
 * information about behavior and return values.
 *
 * Returns:
 * The GPIO descriptor corresponding to the function @con_id of device
 * dev, %-ENOENT if no GPIO has been assigned to the requested function, or
 * another IS_ERR() code if an error occurred while trying to acquire the GPIO.
 */
struct gpio_desc *__must_check devm_gpiod_get(struct device *dev,
                                              const char *con_id,
                                              enum gpiod_flags flags)
{
        return devm_gpiod_get_index(dev, con_id, 0, flags);
}
EXPORT_SYMBOL_GPL(devm_gpiod_get);

/**
 * devm_gpiod_get_optional - Resource-managed gpiod_get_optional()
 * @dev: GPIO consumer
 * @con_id: function within the GPIO consumer
 * @flags: optional GPIO initialization flags
 *
 * Managed gpiod_get_optional(). GPIO descriptors returned from this function
 * are automatically disposed on driver detach. See gpiod_get_optional() for
 * detailed information about behavior and return values.
 *
 * Returns:
 * The GPIO descriptor corresponding to the function @con_id of device
 * dev, NULL if no GPIO has been assigned to the requested function, or
 * another IS_ERR() code if an error occurred while trying to acquire the GPIO.
 */
struct gpio_desc *__must_check devm_gpiod_get_optional(struct device *dev,
                                                       const char *con_id,
                                                       enum gpiod_flags flags)
{
        return devm_gpiod_get_index_optional(dev, con_id, 0, flags);
}
EXPORT_SYMBOL_GPL(devm_gpiod_get_optional);

/**
 * devm_gpiod_get_index - Resource-managed gpiod_get_index()
 * @dev:        GPIO consumer
 * @con_id:        function within the GPIO consumer
 * @idx:        index of the GPIO to obtain in the consumer
 * @flags:        optional GPIO initialization flags
 *
 * Managed gpiod_get_index(). GPIO descriptors returned from this function are
 * automatically disposed on driver detach. See gpiod_get_index() for detailed
 * information about behavior and return values.
 *
 * Returns:
 * The GPIO descriptor corresponding to the function @con_id of device
 * dev, %-ENOENT if no GPIO has been assigned to the requested function, or
 * another IS_ERR() code if an error occurred while trying to acquire the GPIO.
 */
struct gpio_desc *__must_check devm_gpiod_get_index(struct device *dev,
                                                    const char *con_id,
                                                    unsigned int idx,
                                                    enum gpiod_flags flags)
{
        struct gpio_desc **dr;
        struct gpio_desc *desc;

        desc = gpiod_get_index(dev, con_id, idx, flags);
        if (IS_ERR(desc))
                return desc;

        /*
         * For non-exclusive GPIO descriptors, check if this descriptor is
         * already under resource management by this device.
         */
        if (flags & GPIOD_FLAGS_BIT_NONEXCLUSIVE) {
                struct devres *dres;

                dres = devres_find(dev, devm_gpiod_release,
                                   devm_gpiod_match, &desc);
                if (dres)
                        return desc;
        }

        dr = devres_alloc(devm_gpiod_release, sizeof(struct gpio_desc *),
                          GFP_KERNEL);
        if (!dr) {
                gpiod_put(desc);
                return ERR_PTR(-ENOMEM);
        }

        *dr = desc;
        devres_add(dev, dr);

        return desc;
}
EXPORT_SYMBOL_GPL(devm_gpiod_get_index);

/**
 * devm_fwnode_gpiod_get_index - get a GPIO descriptor from a given node
 * @dev:        GPIO consumer
 * @fwnode:        firmware node containing GPIO reference
 * @con_id:        function within the GPIO consumer
 * @index:        index of the GPIO to obtain in the consumer
 * @flags:        GPIO initialization flags
 * @label:        label to attach to the requested GPIO
 *
 * GPIO descriptors returned from this function are automatically disposed on
 * driver detach.
 *
 * Returns:
 * The GPIO descriptor corresponding to the function @con_id of device
 * dev, %-ENOENT if no GPIO has been assigned to the requested function, or
 * another IS_ERR() code if an error occurred while trying to acquire the GPIO.
 */
struct gpio_desc *devm_fwnode_gpiod_get_index(struct device *dev,
                                              struct fwnode_handle *fwnode,
                                              const char *con_id, int index,
                                              enum gpiod_flags flags,
                                              const char *label)
{
        struct gpio_desc **dr;
        struct gpio_desc *desc;

        dr = devres_alloc(devm_gpiod_release, sizeof(struct gpio_desc *),
                          GFP_KERNEL);
        if (!dr)
                return ERR_PTR(-ENOMEM);

        desc = gpiod_find_and_request(dev, fwnode, con_id, index, flags, label, false);
        if (IS_ERR(desc)) {
                devres_free(dr);
                return desc;
        }

        *dr = desc;
        devres_add(dev, dr);

        return desc;
}
EXPORT_SYMBOL_GPL(devm_fwnode_gpiod_get_index);

/**
 * devm_gpiod_get_index_optional - Resource-managed gpiod_get_index_optional()
 * @dev: GPIO consumer
 * @con_id: function within the GPIO consumer
 * @index: index of the GPIO to obtain in the consumer
 * @flags: optional GPIO initialization flags
 *
 * Managed gpiod_get_index_optional(). GPIO descriptors returned from this
 * function are automatically disposed on driver detach. See
 * gpiod_get_index_optional() for detailed information about behavior and
 * return values.
 *
 * Returns:
 * The GPIO descriptor corresponding to the function @con_id of device
 * dev, %NULL if no GPIO has been assigned to the requested function, or
 * another IS_ERR() code if an error occurred while trying to acquire the GPIO.
 */
struct gpio_desc *__must_check devm_gpiod_get_index_optional(struct device *dev,
                                                             const char *con_id,
                                                             unsigned int index,
                                                             enum gpiod_flags flags)
{
        struct gpio_desc *desc;

        desc = devm_gpiod_get_index(dev, con_id, index, flags);
        if (gpiod_not_found(desc))
                return NULL;

        return desc;
}
EXPORT_SYMBOL_GPL(devm_gpiod_get_index_optional);

/**
 * devm_gpiod_get_array - Resource-managed gpiod_get_array()
 * @dev:        GPIO consumer
 * @con_id:        function within the GPIO consumer
 * @flags:        optional GPIO initialization flags
 *
 * Managed gpiod_get_array(). GPIO descriptors returned from this function are
 * automatically disposed on driver detach. See gpiod_get_array() for detailed
 * information about behavior and return values.
 *
 * Returns:
 * The GPIO descriptors corresponding to the function @con_id of device
 * dev, %-ENOENT if no GPIO has been assigned to the requested function,
 * or another IS_ERR() code if an error occurred while trying to acquire
 * the GPIOs.
 */
struct gpio_descs *__must_check devm_gpiod_get_array(struct device *dev,
                                                     const char *con_id,
                                                     enum gpiod_flags flags)
{
        struct gpio_descs **dr;
        struct gpio_descs *descs;

        dr = devres_alloc(devm_gpiod_release_array,
                          sizeof(struct gpio_descs *), GFP_KERNEL);
        if (!dr)
                return ERR_PTR(-ENOMEM);

        descs = gpiod_get_array(dev, con_id, flags);
        if (IS_ERR(descs)) {
                devres_free(dr);
                return descs;
        }

        *dr = descs;
        devres_add(dev, dr);

        return descs;
}
EXPORT_SYMBOL_GPL(devm_gpiod_get_array);

/**
 * devm_gpiod_get_array_optional - Resource-managed gpiod_get_array_optional()
 * @dev:        GPIO consumer
 * @con_id:        function within the GPIO consumer
 * @flags:        optional GPIO initialization flags
 *
 * Managed gpiod_get_array_optional(). GPIO descriptors returned from this
 * function are automatically disposed on driver detach.
 * See gpiod_get_array_optional() for detailed information about behavior and
 * return values.
 *
 * Returns:
 * The GPIO descriptors corresponding to the function @con_id of device
 * dev, %NULL if no GPIO has been assigned to the requested function,
 * or another IS_ERR() code if an error occurred while trying to acquire
 * the GPIOs.
 */
struct gpio_descs *__must_check
devm_gpiod_get_array_optional(struct device *dev, const char *con_id,
                              enum gpiod_flags flags)
{
        struct gpio_descs *descs;

        descs = devm_gpiod_get_array(dev, con_id, flags);
        if (gpiod_not_found(descs))
                return NULL;

        return descs;
}
EXPORT_SYMBOL_GPL(devm_gpiod_get_array_optional);

/**
 * devm_gpiod_put - Resource-managed gpiod_put()
 * @dev:        GPIO consumer
 * @desc:        GPIO descriptor to dispose of
 *
 * Dispose of a GPIO descriptor obtained with devm_gpiod_get() or
 * devm_gpiod_get_index(). Normally this function will not be called as the GPIO
 * will be disposed of by the resource management code.
 */
void devm_gpiod_put(struct device *dev, struct gpio_desc *desc)
{
        WARN_ON(devres_release(dev, devm_gpiod_release, devm_gpiod_match,
                &desc));
}
EXPORT_SYMBOL_GPL(devm_gpiod_put);

/**
 * devm_gpiod_unhinge - Remove resource management from a gpio descriptor
 * @dev:        GPIO consumer
 * @desc:        GPIO descriptor to remove resource management from
 *
 * Remove resource management from a GPIO descriptor. This is needed when
 * you want to hand over lifecycle management of a descriptor to another
 * mechanism.
 */

void devm_gpiod_unhinge(struct device *dev, struct gpio_desc *desc)
{
        int ret;

        if (IS_ERR_OR_NULL(desc))
                return;
        ret = devres_destroy(dev, devm_gpiod_release,
                             devm_gpiod_match, &desc);
        /*
         * If the GPIO descriptor is requested as nonexclusive, we
         * may call this function several times on the same descriptor
         * so it is OK if devres_destroy() returns -ENOENT.
         */
        if (ret == -ENOENT)
                return;
        /* Anything else we should warn about */
        WARN_ON(ret);
}
EXPORT_SYMBOL_GPL(devm_gpiod_unhinge);

/**
 * devm_gpiod_put_array - Resource-managed gpiod_put_array()
 * @dev:        GPIO consumer
 * @descs:        GPIO descriptor array to dispose of
 *
 * Dispose of an array of GPIO descriptors obtained with devm_gpiod_get_array().
 * Normally this function will not be called as the GPIOs will be disposed of
 * by the resource management code.
 */
void devm_gpiod_put_array(struct device *dev, struct gpio_descs *descs)
{
        WARN_ON(devres_release(dev, devm_gpiod_release_array,
                               devm_gpiod_match_array, &descs));
}
EXPORT_SYMBOL_GPL(devm_gpiod_put_array);

static void devm_gpio_chip_release(void *data)
{
        struct gpio_chip *gc = data;

        gpiochip_remove(gc);
}

/**
 * devm_gpiochip_add_data_with_key() - Resource managed gpiochip_add_data_with_key()
 * @dev: pointer to the device that gpio_chip belongs to.
 * @gc: the GPIO chip to register
 * @data: driver-private data associated with this chip
 * @lock_key: lockdep class for IRQ lock
 * @request_key: lockdep class for IRQ request
 *
 * Context: potentially before irqs will work
 *
 * The gpio chip automatically be released when the device is unbound.
 *
 * Returns:
 * A negative errno if the chip can't be registered, such as because the
 * gc->base is invalid or already associated with a different chip.
 * Otherwise it returns zero as a success code.
 */
int devm_gpiochip_add_data_with_key(struct device *dev, struct gpio_chip *gc, void *data,
                                    struct lock_class_key *lock_key,
                                    struct lock_class_key *request_key)
{
        int ret;

        ret = gpiochip_add_data_with_key(gc, data, lock_key, request_key);
        if (ret < 0)
                return ret;

        return devm_add_action_or_reset(dev, devm_gpio_chip_release, gc);
}
EXPORT_SYMBOL_GPL(devm_gpiochip_add_data_with_key);










































    1 












    3 
    1 




    2 
    1 



    1 





    1 
    1 












   12 


    3 











    9 

    1 

    1 
    1 







    8 
    1 

    1 





   12 



















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2010-2011 EIA Electronics,
//                         Kurt Van Dijck <kurt.van.dijck@eia.be>
// Copyright (c) 2010-2011 EIA Electronics,
//                         Pieter Beyens <pieter.beyens@eia.be>
// Copyright (c) 2017-2019 Pengutronix,
//                         Marc Kleine-Budde <kernel@pengutronix.de>
// Copyright (c) 2017-2019 Pengutronix,
//                         Oleksij Rempel <kernel@pengutronix.de>

/* J1939 Address Claiming.
 * Address Claiming in the kernel
 * - keeps track of the AC states of ECU's,
 * - resolves NAME<=>SA taking into account the AC states of ECU's.
 *
 * All Address Claim msgs (including host-originated msg) are processed
 * at the receive path (a sent msg is always received again via CAN echo).
 * As such, the processing of AC msgs is done in the order on which msgs
 * are sent on the bus.
 *
 * This module doesn't send msgs itself (e.g. replies on Address Claims),
 * this is the responsibility of a user space application or daemon.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/netdevice.h>
#include <linux/skbuff.h>

#include "j1939-priv.h"

static inline name_t j1939_skb_to_name(const struct sk_buff *skb)
{
        return le64_to_cpup((__le64 *)skb->data);
}

static inline bool j1939_ac_msg_is_request(struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        int req_pgn;

        if (skb->len < 3 || skcb->addr.pgn != J1939_PGN_REQUEST)
                return false;

        req_pgn = skb->data[0] | (skb->data[1] << 8) | (skb->data[2] << 16);

        return req_pgn == J1939_PGN_ADDRESS_CLAIMED;
}

static int j1939_ac_verify_outgoing(struct j1939_priv *priv,
                                    struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);

        if (skb->len != 8) {
                netdev_notice(priv->ndev, "tx address claim with dlc %i\n",
                              skb->len);
                return -EPROTO;
        }

        if (skcb->addr.src_name != j1939_skb_to_name(skb)) {
                netdev_notice(priv->ndev, "tx address claim with different name\n");
                return -EPROTO;
        }

        if (skcb->addr.sa == J1939_NO_ADDR) {
                netdev_notice(priv->ndev, "tx address claim with broadcast sa\n");
                return -EPROTO;
        }

        /* ac must always be a broadcast */
        if (skcb->addr.dst_name || skcb->addr.da != J1939_NO_ADDR) {
                netdev_notice(priv->ndev, "tx address claim with dest, not broadcast\n");
                return -EPROTO;
        }
        return 0;
}

int j1939_ac_fixup(struct j1939_priv *priv, struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        int ret;
        u8 addr;

        /* network mgmt: address claiming msgs */
        if (skcb->addr.pgn == J1939_PGN_ADDRESS_CLAIMED) {
                struct j1939_ecu *ecu;

                ret = j1939_ac_verify_outgoing(priv, skb);
                /* return both when failure & when successful */
                if (ret < 0)
                        return ret;
                ecu = j1939_ecu_get_by_name(priv, skcb->addr.src_name);
                if (!ecu)
                        return -ENODEV;

                if (ecu->addr != skcb->addr.sa)
                        /* hold further traffic for ecu, remove from parent */
                        j1939_ecu_unmap(ecu);
                j1939_ecu_put(ecu);
        } else if (skcb->addr.src_name) {
                /* assign source address */
                addr = j1939_name_to_addr(priv, skcb->addr.src_name);
                if (!j1939_address_is_unicast(addr) &&
                    !j1939_ac_msg_is_request(skb)) {
                        netdev_notice(priv->ndev, "tx drop: invalid sa for name 0x%016llx\n",
                                      skcb->addr.src_name);
                        return -EADDRNOTAVAIL;
                }
                skcb->addr.sa = addr;
        }

        /* assign destination address */
        if (skcb->addr.dst_name) {
                addr = j1939_name_to_addr(priv, skcb->addr.dst_name);
                if (!j1939_address_is_unicast(addr)) {
                        netdev_notice(priv->ndev, "tx drop: invalid da for name 0x%016llx\n",
                                      skcb->addr.dst_name);
                        return -EADDRNOTAVAIL;
                }
                skcb->addr.da = addr;
        }
        return 0;
}

static void j1939_ac_process(struct j1939_priv *priv, struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_ecu *ecu, *prev;
        name_t name;

        if (skb->len != 8) {
                netdev_notice(priv->ndev, "rx address claim with wrong dlc %i\n",
                              skb->len);
                return;
        }

        name = j1939_skb_to_name(skb);
        skcb->addr.src_name = name;
        if (!name) {
                netdev_notice(priv->ndev, "rx address claim without name\n");
                return;
        }

        if (!j1939_address_is_valid(skcb->addr.sa)) {
                netdev_notice(priv->ndev, "rx address claim with broadcast sa\n");
                return;
        }

        write_lock_bh(&priv->lock);

        /* Few words on the ECU ref counting:
         *
         * First we get an ECU handle, either with
         * j1939_ecu_get_by_name_locked() (increments the ref counter)
         * or j1939_ecu_create_locked() (initializes an ECU object
         * with a ref counter of 1).
         *
         * j1939_ecu_unmap_locked() will decrement the ref counter,
         * but only if the ECU was mapped before. So "ecu" still
         * belongs to us.
         *
         * j1939_ecu_timer_start() will increment the ref counter
         * before it starts the timer, so we can put the ecu when
         * leaving this function.
         */
        ecu = j1939_ecu_get_by_name_locked(priv, name);

        if (ecu && ecu->addr == skcb->addr.sa) {
                /* The ISO 11783-5 standard, in "4.5.2 - Address claim
                 * requirements", states:
                 *   d) No CF shall begin, or resume, transmission on the
                 *      network until 250 ms after it has successfully claimed
                 *      an address except when responding to a request for
                 *      address-claimed.
                 *
                 * But "Figure 6" and "Figure 7" in "4.5.4.2 - Address-claim
                 * prioritization" show that the CF begins the transmission
                 * after 250 ms from the first AC (address-claimed) message
                 * even if it sends another AC message during that time window
                 * to resolve the address contention with another CF.
                 *
                 * As stated in "4.4.2.3 - Address-claimed message":
                 *   In order to successfully claim an address, the CF sending
                 *   an address claimed message shall not receive a contending
                 *   claim from another CF for at least 250 ms.
                 *
                 * As stated in "4.4.3.2 - NAME management (NM) message":
                 *   1) A commanding CF can
                 *      d) request that a CF with a specified NAME transmit
                 *         the address-claimed message with its current NAME.
                 *   2) A target CF shall
                 *      d) send an address-claimed message in response to a
                 *         request for a matching NAME
                 *
                 * Taking the above arguments into account, the 250 ms wait is
                 * requested only during network initialization.
                 *
                 * Do not restart the timer on AC message if both the NAME and
                 * the address match and so if the address has already been
                 * claimed (timer has expired) or the AC message has been sent
                 * to resolve the contention with another CF (timer is still
                 * running).
                 */
                goto out_ecu_put;
        }

        if (!ecu && j1939_address_is_unicast(skcb->addr.sa))
                ecu = j1939_ecu_create_locked(priv, name);

        if (IS_ERR_OR_NULL(ecu))
                goto out_unlock_bh;

        /* cancel pending (previous) address claim */
        j1939_ecu_timer_cancel(ecu);

        if (j1939_address_is_idle(skcb->addr.sa)) {
                j1939_ecu_unmap_locked(ecu);
                goto out_ecu_put;
        }

        /* save new addr */
        if (ecu->addr != skcb->addr.sa)
                j1939_ecu_unmap_locked(ecu);
        ecu->addr = skcb->addr.sa;

        prev = j1939_ecu_get_by_addr_locked(priv, skcb->addr.sa);
        if (prev) {
                if (ecu->name > prev->name) {
                        j1939_ecu_unmap_locked(ecu);
                        j1939_ecu_put(prev);
                        goto out_ecu_put;
                } else {
                        /* kick prev if less or equal */
                        j1939_ecu_unmap_locked(prev);
                        j1939_ecu_put(prev);
                }
        }

        j1939_ecu_timer_start(ecu);
 out_ecu_put:
        j1939_ecu_put(ecu);
 out_unlock_bh:
        write_unlock_bh(&priv->lock);
}

void j1939_ac_recv(struct j1939_priv *priv, struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_ecu *ecu;

        /* network mgmt */
        if (skcb->addr.pgn == J1939_PGN_ADDRESS_CLAIMED) {
                j1939_ac_process(priv, skb);
        } else if (j1939_address_is_unicast(skcb->addr.sa)) {
                /* assign source name */
                ecu = j1939_ecu_get_by_addr(priv, skcb->addr.sa);
                if (ecu) {
                        skcb->addr.src_name = ecu->name;
                        j1939_ecu_put(ecu);
                }
        }

        /* assign destination name */
        ecu = j1939_ecu_get_by_addr(priv, skcb->addr.da);
        if (ecu) {
                skcb->addr.dst_name = ecu->name;
                j1939_ecu_put(ecu);
        }
}




































































    2 























   33 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Definitions for the RAW-IP module.
 *
 * Version:        @(#)raw.h        1.0.2        05/07/93
 *
 * Author:        Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 */
#ifndef _RAW_H
#define _RAW_H

#include <net/inet_sock.h>
#include <net/protocol.h>
#include <net/netns/hash.h>
#include <linux/hash.h>
#include <linux/icmp.h>

extern struct proto raw_prot;

extern struct raw_hashinfo raw_v4_hashinfo;
bool raw_v4_match(struct net *net, const struct sock *sk, unsigned short num,
                  __be32 raddr, __be32 laddr, int dif, int sdif);

int raw_abort(struct sock *sk, int err);
void raw_icmp_error(struct sk_buff *, int, u32);
int raw_local_deliver(struct sk_buff *, int);

int raw_rcv(struct sock *, struct sk_buff *);

#define RAW_HTABLE_LOG        8
#define RAW_HTABLE_SIZE        (1U << RAW_HTABLE_LOG)

struct raw_hashinfo {
        spinlock_t lock;

        struct hlist_head ht[RAW_HTABLE_SIZE] ____cacheline_aligned;
};

static inline u32 raw_hashfunc(const struct net *net, u32 proto)
{
        return hash_32(net_hash_mix(net) ^ proto, RAW_HTABLE_LOG);
}

static inline void raw_hashinfo_init(struct raw_hashinfo *hashinfo)
{
        int i;

        spin_lock_init(&hashinfo->lock);
        for (i = 0; i < RAW_HTABLE_SIZE; i++)
                INIT_HLIST_HEAD(&hashinfo->ht[i]);
}

#ifdef CONFIG_PROC_FS
int raw_proc_init(void);
void raw_proc_exit(void);

struct raw_iter_state {
        struct seq_net_private p;
        int bucket;
};

static inline struct raw_iter_state *raw_seq_private(struct seq_file *seq)
{
        return seq->private;
}
void *raw_seq_start(struct seq_file *seq, loff_t *pos);
void *raw_seq_next(struct seq_file *seq, void *v, loff_t *pos);
void raw_seq_stop(struct seq_file *seq, void *v);
#endif

int raw_hash_sk(struct sock *sk);
void raw_unhash_sk(struct sock *sk);
void raw_init(void);

struct raw_sock {
        /* inet_sock has to be the first member */
        struct inet_sock   inet;
        struct icmp_filter filter;
        u32                   ipmr_table;
};

#define raw_sk(ptr) container_of_const(ptr, struct raw_sock, inet.sk)

static inline bool raw_sk_bound_dev_eq(struct net *net, int bound_dev_if,
                                       int dif, int sdif)
{
#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
        return inet_bound_dev_eq(READ_ONCE(net->ipv4.sysctl_raw_l3mdev_accept),
                                 bound_dev_if, dif, sdif);
#else
        return inet_bound_dev_eq(true, bound_dev_if, dif, sdif);
#endif
}

#endif        /* _RAW_H */























    5 
    5 











    5 







    5 

    5 

    5 








































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2011 Red Hat, Inc.  All Rights Reserved.
 */

#include "xfs.h"
#include "xfs_fs.h"
#include "xfs_error.h"
#include "xfs_shared.h"
#include "xfs_format.h"
#include "xfs_trans_resv.h"
#include "xfs_mount.h"

/*
 * XFS logging functions
 */
static void
__xfs_printk(
        const char                *level,
        const struct xfs_mount        *mp,
        struct va_format        *vaf)
{
        if (mp && mp->m_super) {
                printk("%sXFS (%s): %pV\n", level, mp->m_super->s_id, vaf);
                return;
        }
        printk("%sXFS: %pV\n", level, vaf);
}

void
xfs_printk_level(
        const char *kern_level,
        const struct xfs_mount *mp,
        const char *fmt, ...)
{
        struct va_format        vaf;
        va_list                        args;
        int                        level;

        va_start(args, fmt);
        vaf.fmt = fmt;
        vaf.va = &args;

        __xfs_printk(kern_level, mp, &vaf);

        va_end(args);

        if (!kstrtoint(kern_level, 0, &level) &&
            level <= LOGLEVEL_ERR &&
            xfs_error_level >= XFS_ERRLEVEL_HIGH)
                xfs_stack_trace();
}

void
_xfs_alert_tag(
        const struct xfs_mount        *mp,
        uint32_t                panic_tag,
        const char                *fmt, ...)
{
        struct va_format        vaf;
        va_list                        args;
        int                        do_panic = 0;

        if (xfs_panic_mask && (xfs_panic_mask & panic_tag)) {
                xfs_alert(mp, "Transforming an alert into a BUG.");
                do_panic = 1;
        }

        va_start(args, fmt);

        vaf.fmt = fmt;
        vaf.va = &args;

        __xfs_printk(KERN_ALERT, mp, &vaf);
        va_end(args);

        BUG_ON(do_panic);
}

void
asswarn(
        struct xfs_mount        *mp,
        char                        *expr,
        char                        *file,
        int                        line)
{
        xfs_warn(mp, "Assertion failed: %s, file: %s, line: %d",
                expr, file, line);
        WARN_ON(1);
}

void
assfail(
        struct xfs_mount        *mp,
        char                        *expr,
        char                        *file,
        int                        line)
{
        xfs_emerg(mp, "Assertion failed: %s, file: %s, line: %d",
                expr, file, line);
        if (xfs_globals.bug_on_assert)
                BUG();
        else
                WARN_ON(1);
}

void
xfs_hex_dump(const void *p, int length)
{
        print_hex_dump(KERN_ALERT, "", DUMP_PREFIX_OFFSET, 16, 1, p, length, 1);
}

void
xfs_buf_alert_ratelimited(
        struct xfs_buf                *bp,
        const char                *rlmsg,
        const char                *fmt,
        ...)
{
        struct xfs_mount        *mp = bp->b_mount;
        struct va_format        vaf;
        va_list                        args;

        /* use the more aggressive per-target rate limit for buffers */
        if (!___ratelimit(&bp->b_target->bt_ioerror_rl, rlmsg))
                return;

        va_start(args, fmt);
        vaf.fmt = fmt;
        vaf.va = &args;
        __xfs_printk(KERN_ALERT, mp, &vaf);
        va_end(args);
}

void
xfs_warn_experimental(
        struct xfs_mount                *mp,
        enum xfs_experimental_feat        feat)
{
        static const struct {
                const char                *name;
                long                        opstate;
        } features[] = {
                [XFS_EXPERIMENTAL_PNFS] = {
                        .opstate        = XFS_OPSTATE_WARNED_PNFS,
                        .name                = "pNFS",
                },
                [XFS_EXPERIMENTAL_SCRUB] = {
                        .opstate        = XFS_OPSTATE_WARNED_SCRUB,
                        .name                = "online scrub",
                },
                [XFS_EXPERIMENTAL_SHRINK] = {
                        .opstate        = XFS_OPSTATE_WARNED_SHRINK,
                        .name                = "online shrink",
                },
                [XFS_EXPERIMENTAL_LARP] = {
                        .opstate        = XFS_OPSTATE_WARNED_LARP,
                        .name                = "logged extended attributes",
                },
                [XFS_EXPERIMENTAL_LBS] = {
                        .opstate        = XFS_OPSTATE_WARNED_LBS,
                        .name                = "large block size",
                },
                [XFS_EXPERIMENTAL_EXCHRANGE] = {
                        .opstate        = XFS_OPSTATE_WARNED_EXCHRANGE,
                        .name                = "exchange range",
                },
                [XFS_EXPERIMENTAL_PPTR] = {
                        .opstate        = XFS_OPSTATE_WARNED_PPTR,
                        .name                = "parent pointer",
                },
                [XFS_EXPERIMENTAL_METADIR] = {
                        .opstate        = XFS_OPSTATE_WARNED_METADIR,
                        .name                = "metadata directory tree",
                },
        };
        ASSERT(feat >= 0 && feat < XFS_EXPERIMENTAL_MAX);
        BUILD_BUG_ON(ARRAY_SIZE(features) != XFS_EXPERIMENTAL_MAX);

        if (xfs_should_warn(mp, features[feat].opstate))
                xfs_warn(mp,
 "EXPERIMENTAL %s feature enabled.  Use at your own risk!",
                                features[feat].name);
}











   32 




   51 





   31 






   42 






   43 












   43 
   42 










   72 










   72 







   72 
   72 










   29 


   36 


   70 







    4 
    7 

   23 


   36 


   67 






   58 


   32 


   37 


   73 











   22 













    2 


    1 




    2 


    2 
















   42 



   71 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _XFRM_HASH_H
#define _XFRM_HASH_H

#include <linux/xfrm.h>
#include <linux/socket.h>
#include <linux/jhash.h>

static inline unsigned int __xfrm4_addr_hash(const xfrm_address_t *addr)
{
        return ntohl(addr->a4);
}

static inline unsigned int __xfrm6_addr_hash(const xfrm_address_t *addr)
{
        return jhash2((__force u32 *)addr->a6, 4, 0);
}

static inline unsigned int __xfrm4_daddr_saddr_hash(const xfrm_address_t *daddr,
                                                    const xfrm_address_t *saddr)
{
        u32 sum = (__force u32)daddr->a4 + (__force u32)saddr->a4;
        return ntohl((__force __be32)sum);
}

static inline unsigned int __xfrm6_daddr_saddr_hash(const xfrm_address_t *daddr,
                                                    const xfrm_address_t *saddr)
{
        return __xfrm6_addr_hash(daddr) ^ __xfrm6_addr_hash(saddr);
}

static inline u32 __bits2mask32(__u8 bits)
{
        u32 mask32 = 0xffffffff;

        if (bits == 0)
                mask32 = 0;
        else if (bits < 32)
                mask32 <<= (32 - bits);

        return mask32;
}

static inline unsigned int __xfrm4_dpref_spref_hash(const xfrm_address_t *daddr,
                                                    const xfrm_address_t *saddr,
                                                    __u8 dbits,
                                                    __u8 sbits)
{
        return jhash_2words(ntohl(daddr->a4) & __bits2mask32(dbits),
                            ntohl(saddr->a4) & __bits2mask32(sbits),
                            0);
}

static inline unsigned int __xfrm6_pref_hash(const xfrm_address_t *addr,
                                             __u8 prefixlen)
{
        unsigned int pdw;
        unsigned int pbi;
        u32 initval = 0;

        pdw = prefixlen >> 5;     /* num of whole u32 in prefix */
        pbi = prefixlen &  0x1f;  /* num of bits in incomplete u32 in prefix */

        if (pbi) {
                __be32 mask;

                mask = htonl((0xffffffff) << (32 - pbi));

                initval = (__force u32)(addr->a6[pdw] & mask);
        }

        return jhash2((__force u32 *)addr->a6, pdw, initval);
}

static inline unsigned int __xfrm6_dpref_spref_hash(const xfrm_address_t *daddr,
                                                    const xfrm_address_t *saddr,
                                                    __u8 dbits,
                                                    __u8 sbits)
{
        return __xfrm6_pref_hash(daddr, dbits) ^
               __xfrm6_pref_hash(saddr, sbits);
}

static inline unsigned int __xfrm_dst_hash(const xfrm_address_t *daddr,
                                           const xfrm_address_t *saddr,
                                           u32 reqid, unsigned short family,
                                           unsigned int hmask)
{
        unsigned int h = family ^ reqid;
        switch (family) {
        case AF_INET:
                h ^= __xfrm4_daddr_saddr_hash(daddr, saddr);
                break;
        case AF_INET6:
                h ^= __xfrm6_daddr_saddr_hash(daddr, saddr);
                break;
        }
        return (h ^ (h >> 16)) & hmask;
}

static inline unsigned int __xfrm_src_hash(const xfrm_address_t *daddr,
                                           const xfrm_address_t *saddr,
                                           unsigned short family,
                                           unsigned int hmask)
{
        unsigned int h = family;
        switch (family) {
        case AF_INET:
                h ^= __xfrm4_daddr_saddr_hash(daddr, saddr);
                break;
        case AF_INET6:
                h ^= __xfrm6_daddr_saddr_hash(daddr, saddr);
                break;
        }
        return (h ^ (h >> 16)) & hmask;
}

static inline unsigned int
__xfrm_spi_hash(const xfrm_address_t *daddr, __be32 spi, u8 proto,
                unsigned short family, unsigned int hmask)
{
        unsigned int h = (__force u32)spi ^ proto;
        switch (family) {
        case AF_INET:
                h ^= __xfrm4_addr_hash(daddr);
                break;
        case AF_INET6:
                h ^= __xfrm6_addr_hash(daddr);
                break;
        }
        return (h ^ (h >> 10) ^ (h >> 20)) & hmask;
}

static inline unsigned int
__xfrm_seq_hash(u32 seq, unsigned int hmask)
{
        unsigned int h = seq;
        return (h ^ (h >> 10) ^ (h >> 20)) & hmask;
}

static inline unsigned int __idx_hash(u32 index, unsigned int hmask)
{
        return (index ^ (index >> 8)) & hmask;
}

static inline unsigned int __sel_hash(const struct xfrm_selector *sel,
                                      unsigned short family, unsigned int hmask,
                                      u8 dbits, u8 sbits)
{
        const xfrm_address_t *daddr = &sel->daddr;
        const xfrm_address_t *saddr = &sel->saddr;
        unsigned int h = 0;

        switch (family) {
        case AF_INET:
                if (sel->prefixlen_d < dbits ||
                    sel->prefixlen_s < sbits)
                        return hmask + 1;

                h = __xfrm4_dpref_spref_hash(daddr, saddr, dbits, sbits);
                break;

        case AF_INET6:
                if (sel->prefixlen_d < dbits ||
                    sel->prefixlen_s < sbits)
                        return hmask + 1;

                h = __xfrm6_dpref_spref_hash(daddr, saddr, dbits, sbits);
                break;
        }
        h ^= (h >> 16);
        return h & hmask;
}

static inline unsigned int __addr_hash(const xfrm_address_t *daddr,
                                       const xfrm_address_t *saddr,
                                       unsigned short family,
                                       unsigned int hmask,
                                       u8 dbits, u8 sbits)
{
        unsigned int h = 0;

        switch (family) {
        case AF_INET:
                h = __xfrm4_dpref_spref_hash(daddr, saddr, dbits, sbits);
                break;

        case AF_INET6:
                h = __xfrm6_dpref_spref_hash(daddr, saddr, dbits, sbits);
                break;
        }
        h ^= (h >> 16);
        return h & hmask;
}

struct hlist_head *xfrm_hash_alloc(unsigned int sz);
void xfrm_hash_free(struct hlist_head *n, unsigned int sz);

#endif /* _XFRM_HASH_H */




















































































































































































































   24 




    8 














    7 




    7 




    1 






    1 
    7 




  336 





  648 





  247 





























































































  655 

  655 


  220 











    9 

    1 


    8 
    2 



    7 
    9 






   75 



   65 





   63 








   13 
    1 
   11 




   13 




   34 































   13 




    6 





  173 

  172 
  172 
  172 
  164 



  164 
   12 









  173 
  173 
  173 
  173 
  173 
  173 




































  337 





  274 





  376 




   27 






  121 








  173 

  173 

  153 







  225 

    4 




   11 





   10 

    6 













    7 




    7 







    7 

    2 



    6 





    7 






    5 














    2 

































































    2 
















    2 


    2 




    2 



    2 




    2 

















    2 






































































    1 




















    1 

    7 
    1 

    6 














  171 
  171 


    2 
    1 

    2 




    2 
    2 



    2 



    2 

    2 










    2 
















    3 





    1 
















    2 














    1 









































































































































































































































































































































   54 




    5 
   16 




   13 
    4 















   26 

    1 





   27 















  193 


  100 


  100 






  100 

  194 












  181 


  181 







  181 
  181 


  181 




  193 




  194 
















  191 










   17 








  172 






  173 

  173 



  172 



  171 












   17 


   17 




   17 






   17 


   17 














































   12 

    1 

   12 



























































  596 

   31 

  596 









  598 
  597 

  596 

  594 









  191 


  194 








  193 
    1 
    1 

  194 
















  192 

  191 


  193 

  194 



  193 




  594 




    6 
  313 








   66 


   68 
   68 

































































































































    3 

   65 




   65 

    1 









    3 
    3 






    3 
    3 






  107 
  403 






  403 
    3 




   61 











  594 







  155 



  153 
  118 
  154 
  128 
  154 
  128 
  154 




  595 






  154 







  153 








  155 





  156 
  156 













































  593 








   11 



























  597 






  597 



















   13 
    2 











    2 

    2 

    1 

    1 




   13 














   45 




  196 





   12 











    7 
    9 
    9 






    8 




















  222 







  355 



























  355 



























  356 



  354 



  254 



  354 

  354 
  351 



  352 
  255 
















  356 
    8 
    2 


  355 
   27 
   20 





  356 
  253 






  355 
  254 










































































  357 






  357 
  354 






















  356 












  255 







































  357 
  244 











  357 
  357 


















  355 

  357 
  357 














  489 
  488 
  488 





  485 
  348 
  346 

  347 
  348 





  487 
  348 
  348 

  347 
  347 





  488 

    8 








    8 


















   21 
   21 
   27 

   16 
    7 










  598 















   32 






  570 






  572 
























    1 






  598 






  598 
























  598 



  597 






  597 




















































































   41 






   58 


  648 




   58 
   35 



   37 

  644 







  643 
   43 

   36 


  648 

  645 


  643 
    1 


  644 


  645 




  646 


  646 




  643 









    2 







    2 

    2 
































































































































































































































    1 
    1 






































































































































  411 






















  643 
  648 
  648 






  647 
  648 



   46 
    1 



    1 
   42 

    1 

  646 



  643 























  641 


   76 
   76 







  642 






  642 
  644 


  289 
  276 
   14 


  644 
  443 
   11 
  640 

  641 

  235 
    9 

  229 
  236 
    1 
  644 


  644 




  642 
    3 

    3 



  639 

   40 












  556 




  137 

  126 








  125 
  126 











  125 


  122 

  126 




  135 
  126 
  555 

































































  586 






  511 
  587 






  236 




  231 






  581 


  577 
  173 


  572 


  568 
  306 
  306 




  477 




  186 

  155 


  454 



  404 

























  252 



  118 

  138 

   16 
   10 


    6 




  371 











  234 





  552 




  140 







  583 




    1 



  585 





  560 
   14 



  562 


  557 
  552 

  588 



  580 
    1 


  579 
  583 

  418 







  227 
  186 


























  598 





























   37 
   37 

















   28 









   28 








   24 



   24 
















  340 



   38 



   38 



   38 
  337 


  336 
  338 



  339 



  337 














  300 

























   63 

  299 



  298 

  114 

  289 



   21 




   21 







   21 




















   60 
  308 











    1 





    1 




    1 





  299 




  164 




  299 




  176 




    9 




   27 




    6 




   24 




   60 




    1 




  291 




   24 














   22 


   22 
   22 














   22 


    4 
    4 













  299 



  300 


   24 
   20 
   24 

   24 
    8 





   24 


   22 
   22 


   22 

    4 

















  298 





  300 

  133 






  300 
   22 















  169 

  163 





  162 




  150 

   40 







  119 


   55 




    9 
    9 




  260 
  155 

   58 







   50 








  251 
   69 






   67 





   26 

    6 
  243 
  107 


    1 
    1 



    1 


























    1 







    1 





    1 
    1 
    1 



    1 



  107 























  106 




   33 


   32 








   95 










    1 


    1 













    1 
    1 
    1 


    1 



    1 




   94 
   54 







   42 









  275 






  140 






  297 






























































   13 
    2 









   13 
    8 
    8 
    8 
    8 
    8 

    7 

    1 



    8 
    8 


    8 

    6 


    1 












  127 
  127 
  126 
  127 


  115 

  127 
   75 


   69 

   62 





























































































  300 
  323 












  308 







  298 




  299 


  308 




  300 




   22 



  300 

















  322 









  297 



  300 
  300 








  300 






  300 


  297 











   63 



   60 
   59 

   60 
   60 

    9 


   59 
   48 

   60 


   60 
    7 
    7 






    7 
    1 



    6 
    6 

    6 

   60 
    6 










   60 



   60 







    7 
    7 








  178 







   61 




   26 



























  180 





  144 
  142 





   57 








  135 


   43 


  191 


  148 





    2 



  148 




   43 





  197 














  157 

  145 

  157 
  157 


  157 
   26 




   71 




  119 






  172 
  171 


  171 











  172 























   20 
    2 

    1 



  170 

  138 
   71 
   12 

    1 
   20 







   20 
   20 


  172 



  172 
  172 
  139 


  117 


  117 
  118 



   71 
   42 
   31 





   26 

   25 
    1 



   24 



   24 




   53 


   26 
   26 









   54 
   19 


   54 
   51 






   11 







   54 




  157 
































    5 











    1 
    1 
    4 


    5 


    5 





    5 

    5 



    5 


    5 





















    5 
    2 


    2 






    5 



    4 











    5 




    5 

    5 

    1 



















   21 






   21 

   21 



    1 

   21 



    1 


   20 

   21 


























   54 





   35 
   22 

   35 
   29 
    1 




   28 



   27 




   21 








   16 



    6 
    6 

    5 


    3 

    3 



    3 






    6 
    3 
    1 

    6 




    6 



   27 
    7 

    4 





    4 






    1 



    3 

   24 
   25 




   13 

    1 



   24 

   21 



   26 
















  518 



















    2 











    1 



















  173 









   18 
















   50 
   54 
   54 






    2 




















  171 




  172 
  172 





    5 









   10 

    3 

    8 
    1 





    1 




   10 



    3 



   38 


   38 


    3 






    1 





    1 




    1 











   22 
















   21 





   22 


    2 








   20 




   20 

   22 
















  395 

    1 



  394 

    1 



  394 
    1 







  394 






















































































  226 
  216 
































































    3 








































































    3 























   17 











   16 


   17 




















































    7 

    7 







    1 










    6 
    2 

    4 
    4 















   15 












    1 




   16 
   16 


    1 
   16 









   15 










  271 







  272 







  255 






  255 


  256 
  255 



   17 






    4 
    3 
    1 










    6 










    6 




    6 












    6 
    5 




    1 







  488 











   21 

   21 




   22 






   22 







   21 







































   13 











   17 



    1 












    1 

    1 
















  441 


  184 
  442 

  178 

    1 














  439 








   17 





































  442 












  371 







































  380 





























  379 






  381 





  380 

    2 


  329 
    2 





  379 

    1 





  379 
  381 


  381 




















  381 


   24 




   24 





   24 














   24 
   17 

   24 


   24 

    1 










  378 







  377 
    8 




  379 

   22 
   21 
   22 








  380 



  372 
  378 





  378 
  372 









  381 
  381 
  381 
  381 
  381 




  379 
  379 
    2 





























    1 




    2 


















    3 



    2 
    2 


























   44 
   51 










  210 




   23 





  211 







   95 

  212 




    6 
    9 

    3 














   13 


   10 


    4 

    1 
    3 
    2 


    1 

    4 








   10 






    1 
    4 
    3 
    2 


    1 








    1 








    9 




    2 




    3 














    4 

    2 

    2 

    4 






    4 






    4 



    3 






    3 








    3 

















   15 
   15 
   13 



















    2 


    1 


    1 























































































































































































































































































    1 









    1 








































    1 















  174 


   16 

  174 














  173 











  173 
  173 


    5 
    5 
    1 



    5 



  174 
  173 

  173 






    2 
    1 


    1 











  174 























  439 



  441 




  442 




  440 













  435 


    7 
  271 
    1 


    6 


    5 


    5 

    5 

    3 
    5 
    5 



    5 
    4 
    4 


    4 

    3 





    1 


  434 
    5 







    5 





    3 
    2 
    1 



    4 

    3 
    1 
  431 

  272 




   35 
   27 
    1 
    3 


  273 



  272 

  256 

  256 





  166 





  168 


   75 


    9 





   76 





  236 


  203 

  173 




  173 


  172 

   32 
   16 
    1 


    4 
    3 




   15 
   15 
   15 
   16 
    4 
    2 





    4 
    1 
    1 
   12 
    7 




    6 

    5 
    5 
    3 


    2 



    2 
    1 

    1 





    1 

    1 











    1 
    2 
    2 

    1 




  237 
    4 
  219 

  212 

    8 












   25 












    1 
   25 


   24 





   24 




   24 



   24 



    6 



    6 
    1 




   23 




   22 
   22 
   22 
   21 
   21 


    1 



   21 
   12 


    7 


   12 













   20 


   11 




   19 






   19 




















   78 







    9 
    1 






   78 




   77 
   76 






    2 
















    2 



   56 













   21 
   21 



   21 

   21 




   21 

   79 


   61 


   57 






   58 


   58 
    6 
    6 

    1 




   54 
   53 
    3 
   52 
    1 
    1 
    1 




    2 
    1 


    1 










   57 
















   81 





    1 










    4 

    3 


    1 






    1 


















   81 


























    3 
    2 


    1 




























   54 









   54 
    1 




   53 
    1 




   52 
    1 



   50 

   54 
   48 



















































































    2 














    2 

    2 


    2 























































    2 





    1 










    1 

    1 

















































    1 


    1 




























































    5 






    5 





    5 



















    5 


    3 
    1 




    2 
    2 





    2 







    2 
    2 



























































































































































    6 
    6 



    3 
    6 
    6 



    6 







    2 



    2 

    3 

    3 

    2 









    3 
    3 
    3 


    3 
    3 

    3 


    3 
    2 
    2 


    2 





    3 


















































































































































  226 






   73 




  161 




  162 






    8 





    8 


    2 


    1 




    2 




























































































































  176 






















  176 
   37 
  176 
   12 
  175 
   61 


  175 




   25 
  176 



  176 



   17 

   10 
   17 
  175 


  162 














































































































  163 














    8 






    6 







  161 
































  162 































    5 








































   10 








   10 
    1 
   10 


    9 




    9 





    9 




    9 






    9 

    1 











    5 










    5 

    5 


    5 






    5 

    5 





    5 

    5 
    5 
    5 



    1 






    1 




    1 





















  181 



  180 
   86 
    1 






  177 
    2 
    1 



  176 
    8 




  176 






  176 

    4 

  176 



  163 




  161 
  162 
    2 








    2 
    1 
    2 






    6 




    6 



    6 


  160 
    8 
    1 




    8 


  160 


   76 




















   76 







   22 








   22 



   22 
    5 

    4 








    6 





    6 




    6 
    6 












    2 



    2 
    1 


    1 
    1 








    1 










   60 

    4 
    4 



    3 
    2 



   26 
   26 




   34 
   34 




    5 




   15 
    1 



   14 






   10 





    1 










    3 


    3 

    3 




    3 






    1 














    1 






    8 





   12 



   69 

    8 



    2 
    1 
    1 
    1 




   16 








    1 



    1 



    1 





    3 




    3 







    1 



    1 





    2 



    2 
    2 
    2 






    2 

    2 
    4 

    3 



    1 





    2 




























    2 








   25 



    8 









    1 








    1 







    1 




    1 



    3 


















    4 










    1 





    2 
    1 




    2 
    1 















    1 




    1 








   13 








   44 

  226 
    4 
  226 
    4 
  225 
    4 
  226 











   72 
  226 



   73 














  226 
  226 
  226 
  226 
  225 
  225 









  221 
  226 
    7 
  225 
    3 
  220 

    6 
    6 








  226 
  226 
  226 













   12 


























   12 

    5 






    1 
   12 

   12 
   12 













    6 




    6 



    6 

























   33 




















   34 
    2 




   32 





   34 



   34 















   34 

   34 










































































































   34 




















   34 
















































































   34 





   34 



   34 















































   34 



   33 


   34 


   19 













































   34 
   33 
















































































































































































    2 









    2 







  162 




   32 














    1 



   31 

























   31 






   31 



   31 


   16 






   31 
   31 































  136 





    7 






    7 











  137 









  174 


  158 
  154 
  154 
  152 
  153 
  150 



   41 
    1 







   32 
    2 
    1 


    1 



   40 

   40 
    2 










  204 






    9 





    8 
    8 

    8 
    1 



    7 


    7 
    7 
    1 
    1 










  423 


  415 
    2 

    2 



    2 






  423 




  423 

    2 



  421 




  415 




  414 




  422 







    7 








    6 











    6 





    5 
    1 






    2 


    2 






    2 



    1 






  452 





















































  261 


  261 






  251 





  249 






  250 

    1 
  248 



   13 





  237 
    8 



  229 
    2 



  227 
    1 




  226 
   14 





  225 


  226 






  225 










  226 










  226 










  225 

  204 
  226 
   62 



  180 



  204 






  178 
   20 





  178 







    6 




    1 
    6 



















    6 
    6 






    6 






  178 

    8 







    5 
    1 
















    7 




















    2 
    2 





    2 





























    5 




























































    5 




  172 




  174 








  137 


   14 



   20 










   20 
   13 
    3 
    1 


   20 



    4 




    1 



    1 







   14 















































    1 










    1 





    1 


    1 
    1 














  168 
   32 

  173 








  173 


  173 
   17 




   17 




  173 



  172 



  142 

    3 



    3 










    3 


  139 
    4 

  147 
    2 

    1 


  161 
   12 
















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  174 




  174 
    1 




  174 
    1 




  174 
    1 




  175 
    1 




















    8 






    7 












    7 








    1 








    5 
    2 
    4 



    7 







    5 

    1 


    5 




  205 















   16 
   16 
   16 

   14 

    8 



   16 










  172 
  170 



    8 






  171 





    8 


    8 
    8 
    3 


    8 
    1 


    8 



    6 



    4 


    6 

    6 
    6 





    2 


    6 











    2 











    2 
    1 


    1 


    1 

   21 










  598 









    3 
    3 







    1 



    1 



    1 












    3 



















    2 







   19 
    1 











  167 





  169 

   20 

   20 
   20 


























  179 








  180 





  180 
  179 



    1 
  180 




    7 
    6 



    1 





  177 
    1 




  176 














    1 




    1 



    1 







  174 



  174 



  174 

   17 
  172 


    2 


    2 




  169 


  165 

















   16 
   16 



   16 
   16 
    1 


   16 



    6 





   14 

    1 







   13 
    1 



   12 


    5 












   11 
   12 




   12 





    1 
   12 





   11 


    4 







    1 




    1 



  169 

  169 


  168 
  167 


    1 













   61 
   60 
   15 


   61 
   52 
   27 












   61 
   61 
    9 


   59 
   51 
   24 












   21 
   19 

   12 


   21 





    8 
   21 












   21 

    7 


   21 





    9 
   21 













    7 

    9 


   15 
    5 
    5 
    5 





   14 













    6 

   10 


   15 
   13 
   13 
   13 





   15 












    1 
    5 





    4 









    4 



















    4 









    4 















    8 
   16 





    9 









    9 












    8 






    8 









    9 














    2 
    3 



    1 









    1 












    3 




    1 









    1 






   12 








   15 
   23 


   12 










    2 

    2 



















   23 
    8 

   14 

    9 
    9 




   14 
    9 
   23 

   19 











   23 
   22 









    2 





















    2 
    2 

























    1 


    1 
    1 












    1 




    1 
    1 





















    1 
    1 
























   35 

   34 


   88 
   89 
   87 



  112 















   30 


















  112 
   18 



   98 
   70 


















   68 

   61 
   61 
   61 


   21 
   21 



   15 
   15 
   15 


    5 
    5 
    4 


   16 
   16 
   16 


    3 
    3 
    4 


   24 
    2 

   23 


    3 
    2 

    1 


    2 
    1 

    1 






   94 
   32 
   97 









  218 




    3 
















  221 


  220 


  220 
   62 

   11 




    2 
    2 




    1 






    9 


    9 


   56 

   30 


   30 

   30 

    5 







  211 



  173 









   31 




  112 











  112 
  102 
   47 
   25 

   25 






   11 


   20 







  109 







  338 



  335 

   13 
   13 
   13 
    1 



   10 
   10 


    1 





   21 



   21 






   20 



  335 

  308 
  219 
   28 
   28 



  194 
    2 
    1 


    1 
    1 



  195 
  195 
    1 





  215 



  275 






  306 



  306 
  215 



  194 







  195 



  191 
  190 




    8 




    7 


    5 

    3 
    5 


    5 



    2 




   28 




   27 
   10 
    4 


    4 
    4 






    4 



    6 


    4 
    6 


    6 

    6 


   17 


   27 







  275 



  210 


   64 



  224 





  224 
   63 
    2 
    1 



   63 



  214 
    4 






  254 



  219 
   18 
    1 



  210 
   32 
   27 

   26 
    1 





  219 
  219 




  338 











   18 
    9 



   16 







   15 
    5 
















































   16 












  125 










  125 




   52 
   19 

   46 

   39 

   39 




   33 
   34 

   34 
   32 







   18 
    6 

   18 

   16 

   16 




   12 
   12 

   11 
   11 




   22 



   22 

   21 

   20 



   24 

   23 



   44 

   37 



   18 

   14 



   11 

    5 



   22 

   16 



    7 

    5 



   35 

   29 



    6 

    5 


































   19 

   19 

   13 




   14 


    5 





    3 

    3 






    1 
    1 















  186 
   20 

  183 



   86 
   32 



   90 


   76 





   10 
   10 


    8 










  125 































   82 








   68 






   82 

   48 
   12 












   40 













   49 

   49 





   28 











    2 
    9 
    1 
    9 
    1 
    9 
    1 


   14 
   28 
    1 
   28 
    8 
   28 
    1 



    9 

    9 

    5 










    7 


    1 


    2 



    9 

    9 

    5 








   43 
   20 


   30 




   43 
   20 


   30 




   40 
   16 


   27 




   39 
   17 


   27 














   82 












  148 
  147 


   82 









   82 
   82 
   80 

   81 





   13 
   13 









   13 

   13 
   13 


   13 
   13 










   13 

   13 






   13 










   13 




   12 




   12 

   13 














   85 


   78 

   20 
   20 

    3 

    6 

    5 
    9 
    3 
    4 


    5 

    3 





   12 
   12 

    3 

    2 

    1 
    7 
    1 
    2 


    2 

    1 





   19 
   19 

    3 

    3 

    2 
   15 
   11 
    2 


   13 

   11 





   17 
   17 

    1 

    4 

    3 
   11 
    2 
    3 


    2 

    1 











   37 




   37 


   37 

   37 



    1 














   36 
   37 
   37 
   36 
   37 
   37 

   37 
   14 


   12 










   37 




   37 
   37 
   37 
    8 
   37 

   28 

   28 
    2 


   27 





    2 











    2 
    2 
   37 







  189 

  186 














  191 

    7 


    7 
    7 
    1 


    6 






    6 

    3 
    6 
  188 







  187 

  102 
    1 




  100 



  103 
  103 
   85 





  159 



  157 





  186 
  186 




  106 
   99 
   47 
   44 
   40 
  100 



  103 




   63 

   13 

   63 
   14 
   63 






   76 


    8 

   77 
   14 








  148 
   13 
  148 
   37 
  147 
   28 




  147 



  147 




   86 






  116 







  148 


  146 
   87 
   13 
   13 


  147 
   37 
   37 















  148 
   78 
   29 
    4 
    4 

    4 










    2 
    2 
    1 

    2 







    2 
   78 
   26 



   13 



  126 

  135 




  148 
   30 






  395 









  395 
    1 



  396 



  395 

  395 





























  131 
    1 
    1 



    1 












  129 




   41 



   41 



  102 

  102 










    8 


























    8 




    8 
    1 



    7 




    7 
    7 
    2 
    1 












    6 



    6 
    1 




    5 

    5 




    5 





    5 







    5 








  413 





  413 





  414 







  411 












  410 



  414 
    1 



  413 








  413 








  409 

   19 
   18 
   17 
   17 
   16 
   17 
   17 
   17 

    2 
   17 

    4 


   18 
    1 










   33 



















    9 






























   11 










  163 
    1 




  161 



  162 
    9 

    9 






  153 
    1 
    1 







   16 




























































  606 









    7 




  302 




    2 




   27 














  606 


  607 
  607 

  233 


  609 







  606 



  605 

  607 



  608 

  194 
   42 

    1 



  187 

  184 














  273 
  270 



  272 



  273 
   36 























  245 

  246 
  246 
  200 







    6 

    4 


    6 








































































































  272 









  262 


  246 
  245 
    5 



  272 






    5 





  245 



  223 










  210 

  208 



    2 
    1 
    1 


    1 


    1 





    1 

    1 



  210 


















    1 


    1 
    1 




  595 




  597 


  599 
  598 



  130 


  595 
  599 


  271 












  606 


  404 
    2 


  606 


  609 


  606 




  274 





    4 









  272 
    2 




  264 
   16 
  272 



















  273 


   36 


   36 
   27 

    9 


   36 



   36 






  209 

    7 

  207 



  208 








  606 




  605 



  608 





  606 




  609 
  606 




  608 





    1 







  608 





  603 









  605 


  605 
    1 



  606 
  404 
    1 







  608 


  608 


  607 






  645 
   40 
    2 


   39 
    2 


























   10 





    9 
    9 

    1 



    8 


    8 







    7 

    1 
    1 











    6 







    5 
    1 




    2 
    1 






    4 
    4 
    1 




    4 
    4 



    4 



    2 




    2 













   10 





    3 
    2 









    2 









    2 



    2 





    2 






    2 
    2 











    2 









  374 






    1 
    1 







   10 












    8 


    8 


    8 





    8 




    8 




    8 



    8 

    1 
    1 








    7 















    7 
    7 
    1 






    7 
    1 






    7 
    7 
    1 




    6 
    6 
    5 

    1 
    1 






    5 


    3 
    1 





    2 





    6 
   10 







    4 














    4 
    2 


    2 

    2 



    2 








    2 

    1 









    1 



























  644 
  634 




   20 

    1 
   19 



   19 

   19 











  639 
  625 




   12 
   10 






    4 










   53 
   53 
   52 
   53 
   52 
   52 
   53 
   52 














   49 



   65 
   65 

   65 


   21 
   50 

   49 




   26 
   21 













   36 
   41 

   41 








   62 
   64 






   64 


   65 
   37 



   37 

   36 









   65 



   64 
   65 









































   69 

   65 
   65 

   65 

   69 







   35 
   35 
   33 













   65 


   65 

   65 
   62 
    3 

























   62 


   62 

   54 






   57 

   45 

   45 

























   45 
   41 
   41 








    5 
    5 
    5 
    5 
    5 








   17 




   17 


   17 


   17 
   17 




   16 



   23 


















    8 
   11 
   10 










   13 
    2 

   11 















   61 


   36 


    2 
   16 



   36 






   16 


   15 
   12 





   14 







   13 
   13 
    3 
    3 

   36 







   10 
    1 

   10 







   10 


    9 











    8 


















































   65 


   63 


   65 


   65 





   65 
    4 
    4 

    4 


















































   64 
   65 



   61 







   66 










   64 
   65 

   65 




    6 


   65 


   65 





   61 
   65 

   65 




















   61 

   58 




   58 




























   61 
   61 




   61 
   61 
   60 


   58 
   56 



   60 
   36 
   36 

   36 



















   61 
   61 


   61 
   61 
   52 
   39 

   39 
    4 
    4 

    2 

   39 



   61 
   35 


    9 
    8 
    4 

    4 


    1 

    1 

    4 


   61 
   38 


   61 










   27 






   20 
   26 







   27 


































































    2 
    2 
    1 
    1 


















  313 



  297 

  294 










  311 
  193 


  313 


   69 

   49 
   69 



   69 
   26 




















































   27 























   27 
    4 
    3 
    3 



   27 





   27 
    2 
    2 
    2 
    1 
    1 





















   22 
   22 




























   65 
   65 
   65 
   61 
    1 

   61 










   61 






   60 
   56 
   61 












   47 








   48 
   49 



   17 

   13 
   13 




   13 







    7 


   17 



   51 



   51 
   51 

   49 


  311 











  140 


  140 






  127 


  137 






  139 



  138 


















  139 
  139 
  140 

  140 



  140 
  140 



   85 










    1 



























   61 
    1 





  414 






  414 
   61 







    1 










    1 




  415 




  596 














  578 

  128 



  595 


  596 









  599 


  314 
  138 

  140 

   60 

   10 

    8 


    4 





  599 
  307 




  599 


  599 
   16 

  597 
   13 

   13 





  598 
  140 





   65 
  140 

  140 







  596 

    4 




  599 
  595 
  597 


  336 









  282 



  282 



  284 








  270 
  270 





  584 
   25 


   19 



  177 
    1 




  175 



  173 



  174 








  162 


  573 


   60 
   60 
    1 



   59 



   59 








   53 


  570 
  542 



  259 
  259 
  261 
   35 


  260 

    1 



  260 







  258 
   34 
  250 
  404 





  251 




  172 
  458 
   34 
   34 
   33 
   25 
    8 
    1 




   25 

    8 


  455 
  422 
  422 
  420 
  421 

    2 









  422 

















  420 

   32 






  413 



  401 
  404 
  401 


  380 







  189 




  402 


    8 



  395 
  396 



  396 
  394 









  579 



















































































    2 






    1 



    1 
    1 
    2 





































































    4 

    3 

    2 



    1 
    1 





    2 

























  131 








    1 

  138 

































  139 
    2 
    2 



  137 




  137 
    6 



















    1 




  131 
    5 
    1 



    5 
  135 

























  139 




  139 
  139 


   15 





  139 



  136 
    5 






  136 













  140 

  139 

    2 



  136 











  629 
  625 






  619 
  628 
    9 
  290 
    1 



  628 



  422 


  422 
  423 

    1 
   11 


  423 



  149 
    5 
    5 





  143 
    2 







  143 





  110 



    1 



  141 


    3 
    1 


    2 
    2 




  138 



  140 
  139 

  134 






  132 

   45 


    1 



   44 
    1 



   43 

    1 




   42 







  417 






    1 














  579 






  653 





   74 



   74 
   74 

   74 

   74 











  252 









  252 



  184 






  183 
  184 

  183 










  184 
  184 

  183 





  251 



  252 
    1 













  252 





  252 








  252 
  184 
  252 









    5 




    5 
    5 

    5 


    5 



    5 
    1 





    5 

    1 

    1 


    5 








   36 
   36 


   36 
   11 




   36 
   36 

   36 
    1 





    1 


















   36 
   10 



   35 
   36 







   36 






























































   36 




    2 

   36 
   36 


   36 



   36 



   36 
























    6 
    6 

    2 












  372 


  373 




  372 





  371 
  373 


  127 
   12 
  123 





    1 

   12 





  371 
  370 


  373 



  373 
   12 
  373 


   12 


   12 










  370 




  373 
  371 


  372 


   36 


   36 

   36 





















  373 



  374 
  370 
  373 



  363 











   15 
   15 

   10 




    8 
    2 


    7 


















  226 



    2 


    2 




    2 





    9 


    9 


  370 













  379 

  374 













    1 
















  379 




  115 




  116 
    3 



    3 




  116 
  116 

  379 
  379 



  373 



  374 


















  234 



  375 























  129 
    8 









    8 





  312 

  375 












































  212 
  175 








  211 

  180 


   37 







    4 


   37 



  212 


  211 
   61 




  211 
   33 

   17 



   16 

   17 


   31 



    1 


    2 




    3 




  211 



  211 











   21 






  374 


   21 
   21 






   21 



  373 




   21 



   21 

















   21 




   21 



   21 

   21 










   21 





   21 









    1 


    1 

    1 
    1 


   21 


    1 

   21 










   21 
   21 
   12 
   12 


   20 



   21 



   21 




   21 
   21 




   21 






   21 
   21 
   21 
   21 





   21 

   21 














   21 



   21 
   21 

   21 




   21 






   21 
   21 




   21 
   21 





   21 
   21 





   21 

   21 
   21 
   21 


   21 
















































  374 

    4 





  374 
  372 






    4 



    4 






    4 
    4 







    4 






































































































































































































  374 


  378 






























  374 
  378 











  375 




  378 
  374 
  376 

    8 

    3 

    3 











    3 

  377 


    3 





  378 
  378 
  378 



    2 
    2 

    4 








    4 








    4 
























    4 






















    3 

    1 

    2 
    1 



    4 



    4 






  378 





























  378 


    5 
    5 




    5 



    5 






  377 







   35 




    2 




    2 
    2 
    2 










    2 

    2 

    1 

    1 
    1 

    2 





    2 





  376 
    2 




    2 

    1 








    2 





  377 

  127 



















  126 


  126 
    1 
  126 
   15 
  126 

  126 





    8 
    8 

    8 










    7 

    7 
    6 
    6 
    5 












    5 



    2 








    1 




    1 












    1 





  121 
































  121 



















  121 





















  121 

  121 
  109 
  108 
  107 
  106 
  104 
  104 


   23 


   23 


   10 
    8 


    8 




    8 




    8 






























    3 

    2 





    3 


    2 


    1 


    1 


    4 





    2 







  102 


    5 














    5 







   97 
    4 






    4 











    4 






   94 


















































































   94 
   93 














    1 
































    1 






   93 
















   93 









   92 

  378 
  376 






  378 


  378 
  378 
  374 




    2 





    2 







    2 










  375 
    5 

    5 
    5 




    5 






  376 




















































































  125 















  378 





  376 

  378 

  125 
















  378 
  378 







  378 









  379 







  380 
  381 








  381 
  381 

  379 












  596 

    5 




  594 




  594 
  596 


  133 




  595 





  596 










  594 
  596 







  597 








  595 




































































  595 
    1 

    1 



  594 



  599 




  593 
  594 


  593 
  593 
  349 
  596 

























  381 




  379 


    1 




    1 






































  595 


  379 









  127 



  127 



   19 

  129 

  653 












    2 






    1 



    1 








































































    2 







































    3 













    2 


    1 
    1 




    1 

    1 






























































































































































































































































































    3 























    2 
    1 









    1 






  606 








   23 


    1 



  582 
    1 



  581 
    2 

  602 




    3 

































    3 


























    5 











    2 


    2 

    1 





    1 

  655 





    2 










  658 










    3 
    1 



    1 
    2 
    1 

    2 
    2 








  658 
















  658 



  659 



  658 
  659 
  655 




  659 
  660 






   22 




  637 





  659 



  652 








  653 



  652 


  631 
  653 








  656 
   10 


  641 



  642 



  641 



  629 
  618 


  606 
    4 




  607 



  606 



  599 



  599 


  381 
    4 


  655 





  381 


  378 



  379 



  373 

  373 
  371 
  373 


    6 


  371 

  379 


  377 




  375 
  374 
  374 




  378 

  650 
  654 





  390 

  651 






  637 


  376 

   74 








   74 



  373 













  375 



   74 


  374 


  653 



  579 
  653 
  653 




  652 
    3 

  653 




   22 
  655 


  654 
  654 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235
7236
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386
7387
7388
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
7476
7477
7478
7479
7480
7481
7482
7483
7484
7485
7486
7487
7488
7489
7490
7491
7492
7493
7494
7495
7496
7497
7498
7499
7500
7501
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514
7515
7516
7517
7518
7519
7520
7521
7522
7523
7524
7525
7526
7527
7528
7529
7530
7531
7532
7533
7534
7535
7536
7537
7538
7539
7540
7541
7542
7543
7544
7545
7546
7547
7548
7549
7550
7551
7552
7553
7554
7555
7556
7557
7558
7559
7560
7561
7562
7563
7564
7565
7566
7567
7568
7569
7570
7571
7572
7573
7574
7575
7576
7577
7578
7579
7580
7581
7582
7583
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611
7612
7613
7614
7615
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625
7626
7627
7628
7629
7630
7631
7632
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642
7643
7644
7645
7646
7647
7648
7649
7650
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661
7662
7663
7664
7665
7666
7667
7668
7669
7670
7671
7672
7673
7674
7675
7676
7677
7678
7679
7680
7681
7682
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
7706
7707
7708
7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
7720
7721
7722
7723
7724
7725
7726
7727
7728
7729
7730
7731
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741
7742
7743
7744
7745
7746
7747
7748
7749
7750
7751
7752
7753
7754
7755
7756
7757
7758
7759
7760
7761
7762
7763
7764
7765
7766
7767
7768
7769
7770
7771
7772
7773
7774
7775
7776
7777
7778
7779
7780
7781
7782
7783
7784
7785
7786
7787
7788
7789
7790
7791
7792
7793
7794
7795
7796
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
7810
7811
7812
7813
7814
7815
7816
7817
7818
7819
7820
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
7833
7834
7835
7836
7837
7838
7839
7840
7841
7842
7843
7844
7845
7846
7847
7848
7849
7850
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860
7861
7862
7863
7864
7865
7866
7867
7868
7869
7870
7871
7872
7873
7874
7875
7876
7877
7878
7879
7880
7881
7882
7883
7884
7885
7886
7887
7888
7889
7890
7891
7892
7893
7894
7895
7896
7897
7898
7899
7900
7901
7902
7903
7904
7905
7906
7907
7908
7909
7910
7911
7912
7913
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
7924
7925
7926
7927
7928
7929
7930
7931
7932
7933
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944
7945
7946
7947
7948
7949
7950
7951
7952
7953
7954
7955
7956
7957
7958
7959
7960
7961
7962
7963
7964
7965
7966
7967
7968
7969
7970
7971
7972
7973
7974
7975
7976
7977
7978
7979
7980
7981
7982
7983
7984
7985
7986
7987
7988
7989
7990
7991
7992
7993
7994
7995
7996
7997
7998
7999
8000
8001
8002
8003
8004
8005
8006
8007
8008
8009
8010
8011
8012
8013
8014
8015
8016
8017
8018
8019
8020
8021
8022
8023
8024
8025
8026
8027
8028
8029
8030
8031
8032
8033
8034
8035
8036
8037
8038
8039
8040
8041
8042
8043
8044
8045
8046
8047
8048
8049
8050
8051
8052
8053
8054
8055
8056
8057
8058
8059
8060
8061
8062
8063
8064
8065
8066
8067
8068
8069
8070
8071
8072
8073
8074
8075
8076
8077
8078
8079
8080
8081
8082
8083
8084
8085
8086
8087
8088
8089
8090
8091
8092
8093
8094
8095
8096
8097
8098
8099
8100
8101
8102
8103
8104
8105
8106
8107
8108
8109
8110
8111
8112
8113
8114
8115
8116
8117
8118
8119
8120
8121
8122
8123
8124
8125
8126
8127
8128
8129
8130
8131
8132
8133
8134
8135
8136
8137
8138
8139
8140
8141
8142
8143
8144
8145
8146
8147
8148
8149
8150
8151
8152
8153
8154
8155
8156
8157
8158
8159
8160
8161
8162
8163
8164
8165
8166
8167
8168
8169
8170
8171
8172
8173
8174
8175
8176
8177
8178
8179
8180
8181
8182
8183
8184
8185
8186
8187
8188
8189
8190
8191
8192
8193
8194
8195
8196
8197
8198
8199
8200
8201
8202
8203
8204
8205
8206
8207
8208
8209
8210
8211
8212
8213
8214
8215
8216
8217
8218
8219
8220
8221
8222
8223
8224
8225
8226
8227
8228
8229
8230
8231
8232
8233
8234
8235
8236
8237
8238
8239
8240
8241
8242
8243
8244
8245
8246
8247
8248
8249
8250
8251
8252
8253
8254
8255
8256
8257
8258
8259
8260
8261
8262
8263
8264
8265
8266
8267
8268
8269
8270
8271
8272
8273
8274
8275
8276
8277
8278
8279
8280
8281
8282
8283
8284
8285
8286
8287
8288
8289
8290
8291
8292
8293
8294
8295
8296
8297
8298
8299
8300
8301
8302
8303
8304
8305
8306
8307
8308
8309
8310
8311
8312
8313
8314
8315
8316
8317
8318
8319
8320
8321
8322
8323
8324
8325
8326
8327
8328
8329
8330
8331
8332
8333
8334
8335
8336
8337
8338
8339
8340
8341
8342
8343
8344
8345
8346
8347
8348
8349
8350
8351
8352
8353
8354
8355
8356
8357
8358
8359
8360
8361
8362
8363
8364
8365
8366
8367
8368
8369
8370
8371
8372
8373
8374
8375
8376
8377
8378
8379
8380
8381
8382
8383
8384
8385
8386
8387
8388
8389
8390
8391
8392
8393
8394
8395
8396
8397
8398
8399
8400
8401
8402
8403
8404
8405
8406
8407
8408
8409
8410
8411
8412
8413
8414
8415
8416
8417
8418
8419
8420
8421
8422
8423
8424
8425
8426
8427
8428
8429
8430
8431
8432
8433
8434
8435
8436
8437
8438
8439
8440
8441
8442
8443
8444
8445
8446
8447
8448
8449
8450
8451
8452
8453
8454
8455
8456
8457
8458
8459
8460
8461
8462
8463
8464
8465
8466
8467
8468
8469
8470
8471
8472
8473
8474
8475
8476
8477
8478
8479
8480
8481
8482
8483
8484
8485
8486
8487
8488
8489
8490
8491
8492
8493
8494
8495
8496
8497
8498
8499
8500
8501
8502
8503
8504
8505
8506
8507
8508
8509
8510
8511
8512
8513
8514
8515
8516
8517
8518
8519
8520
8521
8522
8523
8524
8525
8526
8527
8528
8529
8530
8531
8532
8533
8534
8535
8536
8537
8538
8539
8540
8541
8542
8543
8544
8545
8546
8547
8548
8549
8550
8551
8552
8553
8554
8555
8556
8557
8558
8559
8560
8561
8562
8563
8564
8565
8566
8567
8568
8569
8570
8571
8572
8573
8574
8575
8576
8577
8578
8579
8580
8581
8582
8583
8584
8585
8586
8587
8588
8589
8590
8591
8592
8593
8594
8595
8596
8597
8598
8599
8600
8601
8602
8603
8604
8605
8606
8607
8608
8609
8610
8611
8612
8613
8614
8615
8616
8617
8618
8619
8620
8621
8622
8623
8624
8625
8626
8627
8628
8629
8630
8631
8632
8633
8634
8635
8636
8637
8638
8639
8640
8641
8642
8643
8644
8645
8646
8647
8648
8649
8650
8651
8652
8653
8654
8655
8656
8657
8658
8659
8660
8661
8662
8663
8664
8665
8666
8667
8668
8669
8670
8671
8672
8673
8674
8675
8676
8677
8678
8679
8680
8681
8682
8683
8684
8685
8686
8687
8688
8689
8690
8691
8692
8693
8694
8695
8696
8697
8698
8699
8700
8701
8702
8703
8704
8705
8706
8707
8708
8709
8710
8711
8712
8713
8714
8715
8716
8717
8718
8719
8720
8721
8722
8723
8724
8725
8726
8727
8728
8729
8730
8731
8732
8733
8734
8735
8736
8737
8738
8739
8740
8741
8742
8743
8744
8745
8746
8747
8748
8749
8750
8751
8752
8753
8754
8755
8756
8757
8758
8759
8760
8761
8762
8763
8764
8765
8766
8767
8768
8769
8770
8771
8772
8773
8774
8775
8776
8777
8778
8779
8780
8781
8782
8783
8784
8785
8786
8787
8788
8789
8790
8791
8792
8793
8794
8795
8796
8797
8798
8799
8800
8801
8802
8803
8804
8805
8806
8807
8808
8809
8810
8811
8812
8813
8814
8815
8816
8817
8818
8819
8820
8821
8822
8823
8824
8825
8826
8827
8828
8829
8830
8831
8832
8833
8834
8835
8836
8837
8838
8839
8840
8841
8842
8843
8844
8845
8846
8847
8848
8849
8850
8851
8852
8853
8854
8855
8856
8857
8858
8859
8860
8861
8862
8863
8864
8865
8866
8867
8868
8869
8870
8871
8872
8873
8874
8875
8876
8877
8878
8879
8880
8881
8882
8883
8884
8885
8886
8887
8888
8889
8890
8891
8892
8893
8894
8895
8896
8897
8898
8899
8900
8901
8902
8903
8904
8905
8906
8907
8908
8909
8910
8911
8912
8913
8914
8915
8916
8917
8918
8919
8920
8921
8922
8923
8924
8925
8926
8927
8928
8929
8930
8931
8932
8933
8934
8935
8936
8937
8938
8939
8940
8941
8942
8943
8944
8945
8946
8947
8948
8949
8950
8951
8952
8953
8954
8955
8956
8957
8958
8959
8960
8961
8962
8963
8964
8965
8966
8967
8968
8969
8970
8971
8972
8973
8974
8975
8976
8977
8978
8979
8980
8981
8982
8983
8984
8985
8986
8987
8988
8989
8990
8991
8992
8993
8994
8995
8996
8997
8998
8999
9000
9001
9002
9003
9004
9005
9006
9007
9008
9009
9010
9011
9012
9013
9014
9015
9016
9017
9018
9019
9020
9021
9022
9023
9024
9025
9026
9027
9028
9029
9030
9031
9032
9033
9034
9035
9036
9037
9038
9039
9040
9041
9042
9043
9044
9045
9046
9047
9048
9049
9050
9051
9052
9053
9054
9055
9056
9057
9058
9059
9060
9061
9062
9063
9064
9065
9066
9067
9068
9069
9070
9071
9072
9073
9074
9075
9076
9077
9078
9079
9080
9081
9082
9083
9084
9085
9086
9087
9088
9089
9090
9091
9092
9093
9094
9095
9096
9097
9098
9099
9100
9101
9102
9103
9104
9105
9106
9107
9108
9109
9110
9111
9112
9113
9114
9115
9116
9117
9118
9119
9120
9121
9122
9123
9124
9125
9126
9127
9128
9129
9130
9131
9132
9133
9134
9135
9136
9137
9138
9139
9140
9141
9142
9143
9144
9145
9146
9147
9148
9149
9150
9151
9152
9153
9154
9155
9156
9157
9158
9159
9160
9161
9162
9163
9164
9165
9166
9167
9168
9169
9170
9171
9172
9173
9174
9175
9176
9177
9178
9179
9180
9181
9182
9183
9184
9185
9186
9187
9188
9189
9190
9191
9192
9193
9194
9195
9196
9197
9198
9199
9200
9201
9202
9203
9204
9205
9206
9207
9208
9209
9210
9211
9212
9213
9214
9215
9216
9217
9218
9219
9220
9221
9222
9223
9224
9225
9226
9227
9228
9229
9230
9231
9232
9233
9234
9235
9236
9237
9238
9239
9240
9241
9242
9243
9244
9245
9246
9247
9248
9249
9250
9251
9252
9253
9254
9255
9256
9257
9258
9259
9260
9261
9262
9263
9264
9265
9266
9267
9268
9269
9270
9271
9272
9273
9274
9275
9276
9277
9278
9279
9280
9281
9282
9283
9284
9285
9286
9287
9288
9289
9290
9291
9292
9293
9294
9295
9296
9297
9298
9299
9300
9301
9302
9303
9304
9305
9306
9307
9308
9309
9310
9311
9312
9313
9314
9315
9316
9317
9318
9319
9320
9321
9322
9323
9324
9325
9326
9327
9328
9329
9330
9331
9332
9333
9334
9335
9336
9337
9338
9339
9340
9341
9342
9343
9344
9345
9346
9347
9348
9349
9350
9351
9352
9353
9354
9355
9356
9357
9358
9359
9360
9361
9362
9363
9364
9365
9366
9367
9368
9369
9370
9371
9372
9373
9374
9375
9376
9377
9378
9379
9380
9381
9382
9383
9384
9385
9386
9387
9388
9389
9390
9391
9392
9393
9394
9395
9396
9397
9398
9399
9400
9401
9402
9403
9404
9405
9406
9407
9408
9409
9410
9411
9412
9413
9414
9415
9416
9417
9418
9419
9420
9421
9422
9423
9424
9425
9426
9427
9428
9429
9430
9431
9432
9433
9434
9435
9436
9437
9438
9439
9440
9441
9442
9443
9444
9445
9446
9447
9448
9449
9450
9451
9452
9453
9454
9455
9456
9457
9458
9459
9460
9461
9462
9463
9464
9465
9466
9467
9468
9469
9470
9471
9472
9473
9474
9475
9476
9477
9478
9479
9480
9481
9482
9483
9484
9485
9486
9487
9488
9489
9490
9491
9492
9493
9494
9495
9496
9497
9498
9499
9500
9501
9502
9503
9504
9505
9506
9507
9508
9509
9510
9511
9512
9513
9514
9515
9516
9517
9518
9519
9520
9521
9522
9523
9524
9525
9526
9527
9528
9529
9530
9531
9532
9533
9534
9535
9536
9537
9538
9539
9540
9541
9542
9543
9544
9545
9546
9547
9548
9549
9550
9551
9552
9553
9554
9555
9556
9557
9558
9559
9560
9561
9562
9563
9564
9565
9566
9567
9568
9569
9570
9571
9572
9573
9574
9575
9576
9577
9578
9579
9580
9581
9582
9583
9584
9585
9586
9587
9588
9589
9590
9591
9592
9593
9594
9595
9596
9597
9598
9599
9600
9601
9602
9603
9604
9605
9606
9607
9608
9609
9610
9611
9612
9613
9614
9615
9616
9617
9618
9619
9620
9621
9622
9623
9624
9625
9626
9627
9628
9629
9630
9631
9632
9633
9634
9635
9636
9637
9638
9639
9640
9641
9642
9643
9644
9645
9646
9647
9648
9649
9650
9651
9652
9653
9654
9655
9656
9657
9658
9659
9660
9661
9662
9663
9664
9665
9666
9667
9668
9669
9670
9671
9672
9673
9674
9675
9676
9677
9678
9679
9680
9681
9682
9683
9684
9685
9686
9687
9688
9689
9690
9691
9692
9693
9694
9695
9696
9697
9698
9699
9700
9701
9702
9703
9704
9705
9706
9707
9708
9709
9710
9711
9712
9713
9714
9715
9716
9717
9718
9719
9720
9721
9722
9723
9724
9725
9726
9727
9728
9729
9730
9731
9732
9733
9734
9735
9736
9737
9738
9739
9740
9741
9742
9743
9744
9745
9746
9747
9748
9749
9750
9751
9752
9753
9754
9755
9756
9757
9758
9759
9760
9761
9762
9763
9764
9765
9766
9767
9768
9769
9770
9771
9772
9773
9774
9775
9776
9777
9778
9779
9780
9781
9782
9783
9784
9785
9786
9787
9788
9789
9790
9791
9792
9793
9794
9795
9796
9797
9798
9799
9800
9801
9802
9803
9804
9805
9806
9807
9808
9809
9810
9811
9812
9813
9814
9815
9816
9817
9818
9819
9820
9821
9822
9823
9824
9825
9826
9827
9828
9829
9830
9831
9832
9833
9834
9835
9836
9837
9838
9839
9840
9841
9842
9843
9844
9845
9846
9847
9848
9849
9850
9851
9852
9853
9854
9855
9856
9857
9858
9859
9860
9861
9862
9863
9864
9865
9866
9867
9868
9869
9870
9871
9872
9873
9874
9875
9876
9877
9878
9879
9880
9881
9882
9883
9884
9885
9886
9887
9888
9889
9890
9891
9892
9893
9894
9895
9896
9897
9898
9899
9900
9901
9902
9903
9904
9905
9906
9907
9908
9909
9910
9911
9912
9913
9914
9915
9916
9917
9918
9919
9920
9921
9922
9923
9924
9925
9926
9927
9928
9929
9930
9931
9932
9933
9934
9935
9936
9937
9938
9939
9940
9941
9942
9943
9944
9945
9946
9947
9948
9949
9950
9951
9952
9953
9954
9955
9956
9957
9958
9959
9960
9961
9962
9963
9964
9965
9966
9967
9968
9969
9970
9971
9972
9973
9974
9975
9976
9977
9978
9979
9980
9981
9982
9983
9984
9985
9986
9987
9988
9989
9990
9991
9992
9993
9994
9995
9996
9997
9998
9999
10000
10001
10002
10003
10004
10005
10006
10007
10008
10009
10010
10011
10012
10013
10014
10015
10016
10017
10018
10019
10020
10021
10022
10023
10024
10025
10026
10027
10028
10029
10030
10031
10032
10033
10034
10035
10036
10037
10038
10039
10040
10041
10042
10043
10044
10045
10046
10047
10048
10049
10050
10051
10052
10053
10054
10055
10056
10057
10058
10059
10060
10061
10062
10063
10064
10065
10066
10067
10068
10069
10070
10071
10072
10073
10074
10075
10076
10077
10078
10079
10080
10081
10082
10083
10084
10085
10086
10087
10088
10089
10090
10091
10092
10093
10094
10095
10096
10097
10098
10099
10100
10101
10102
10103
10104
10105
10106
10107
10108
10109
10110
10111
10112
10113
10114
10115
10116
10117
10118
10119
10120
10121
10122
10123
10124
10125
10126
10127
10128
10129
10130
10131
10132
10133
10134
10135
10136
10137
10138
10139
10140
10141
10142
10143
10144
10145
10146
10147
10148
10149
10150
10151
10152
10153
10154
10155
10156
10157
10158
10159
10160
10161
10162
10163
10164
10165
10166
10167
10168
10169
10170
10171
10172
10173
10174
10175
10176
10177
10178
10179
10180
10181
10182
10183
10184
10185
10186
10187
10188
10189
10190
10191
10192
10193
10194
10195
10196
10197
10198
10199
10200
10201
10202
10203
10204
10205
10206
10207
10208
10209
10210
10211
10212
10213
10214
10215
10216
10217
10218
10219
10220
10221
10222
10223
10224
10225
10226
10227
10228
10229
10230
10231
10232
10233
10234
10235
10236
10237
10238
10239
10240
10241
10242
10243
10244
10245
10246
10247
10248
10249
10250
10251
10252
10253
10254
10255
10256
10257
10258
10259
10260
10261
10262
10263
10264
10265
10266
10267
10268
10269
10270
10271
10272
10273
10274
10275
10276
10277
10278
10279
10280
10281
10282
10283
10284
10285
10286
10287
10288
10289
10290
10291
10292
10293
10294
10295
10296
10297
10298
10299
10300
10301
10302
10303
10304
10305
10306
10307
10308
10309
10310
10311
10312
10313
10314
10315
10316
10317
10318
10319
10320
10321
10322
10323
10324
10325
10326
10327
10328
10329
10330
10331
10332
10333
10334
10335
10336
10337
10338
10339
10340
10341
10342
10343
10344
10345
10346
10347
10348
10349
10350
10351
10352
10353
10354
10355
10356
10357
10358
10359
10360
10361
10362
10363
10364
10365
10366
10367
10368
10369
10370
10371
10372
10373
10374
10375
10376
10377
10378
10379
10380
10381
10382
10383
10384
10385
10386
10387
10388
10389
10390
10391
10392
10393
10394
10395
10396
10397
10398
10399
10400
10401
10402
10403
10404
10405
10406
10407
10408
10409
10410
10411
10412
10413
10414
10415
10416
10417
10418
10419
10420
10421
10422
10423
10424
10425
10426
10427
10428
10429
10430
10431
10432
10433
10434
10435
10436
10437
10438
10439
10440
10441
10442
10443
10444
10445
10446
10447
10448
10449
10450
10451
10452
10453
10454
10455
10456
10457
10458
10459
10460
10461
10462
10463
10464
10465
10466
10467
10468
10469
10470
10471
10472
10473
10474
10475
10476
10477
10478
10479
10480
10481
10482
10483
10484
10485
10486
10487
10488
10489
10490
10491
10492
10493
10494
10495
10496
10497
10498
10499
10500
10501
10502
10503
10504
10505
10506
10507
10508
10509
10510
10511
10512
10513
10514
10515
10516
10517
10518
10519
10520
10521
10522
10523
10524
10525
10526
10527
10528
10529
10530
10531
10532
10533
10534
10535
10536
10537
10538
10539
10540
10541
10542
10543
10544
10545
10546
10547
10548
10549
10550
10551
10552
10553
10554
10555
10556
10557
10558
10559
10560
10561
10562
10563
10564
10565
10566
10567
10568
10569
10570
10571
10572
10573
10574
10575
10576
10577
10578
10579
10580
10581
10582
10583
10584
10585
10586
10587
10588
10589
10590
10591
10592
10593
10594
10595
10596
10597
10598
10599
10600
10601
10602
10603
10604
10605
10606
10607
10608
10609
10610
10611
10612
10613
10614
10615
10616
10617
10618
10619
10620
10621
10622
10623
10624
10625
10626
10627
10628
10629
10630
10631
10632
10633
10634
10635
10636
10637
10638
10639
10640
10641
10642
10643
10644
10645
10646
10647
10648
10649
10650
10651
10652
10653
10654
10655
10656
10657
10658
10659
10660
10661
10662
10663
10664
10665
10666
10667
10668
10669
10670
10671
10672
10673
10674
10675
10676
10677
10678
10679
10680
10681
10682
10683
10684
10685
10686
10687
10688
10689
10690
10691
10692
10693
10694
10695
10696
10697
10698
10699
10700
10701
10702
10703
10704
10705
10706
10707
10708
10709
10710
10711
10712
10713
10714
10715
10716
10717
10718
10719
10720
10721
10722
10723
10724
10725
10726
10727
10728
10729
10730
10731
10732
10733
10734
10735
10736
10737
10738
10739
10740
10741
10742
10743
10744
10745
10746
10747
10748
10749
10750
10751
10752
10753
10754
10755
10756
10757
10758
10759
10760
10761
10762
10763
10764
10765
10766
10767
10768
10769
10770
10771
10772
10773
10774
10775
10776
10777
10778
10779
10780
10781
10782
10783
10784
10785
10786
10787
10788
10789
10790
10791
10792
10793
10794
10795
10796
10797
10798
10799
10800
10801
10802
10803
10804
10805
10806
10807
10808
10809
10810
10811
10812
10813
10814
10815
10816
10817
10818
10819
10820
10821
10822
10823
10824
10825
10826
10827
10828
10829
10830
10831
10832
10833
10834
10835
10836
10837
10838
10839
10840
10841
10842
10843
10844
10845
10846
10847
10848
10849
10850
10851
10852
10853
10854
10855
10856
10857
10858
10859
10860
10861
10862
10863
10864
10865
10866
10867
10868
10869
10870
10871
10872
10873
10874
10875
10876
10877
10878
10879
10880
10881
10882
10883
10884
10885
10886
10887
10888
10889
10890
10891
10892
10893
10894
10895
10896
10897
10898
10899
10900
10901
10902
10903
10904
10905
10906
10907
10908
10909
10910
10911
10912
10913
10914
10915
10916
10917
10918
10919
10920
10921
10922
10923
10924
10925
10926
10927
10928
10929
10930
10931
10932
10933
10934
10935
10936
10937
10938
10939
10940
10941
10942
10943
10944
10945
10946
10947
10948
10949
10950
10951
10952
10953
10954
10955
10956
10957
10958
10959
10960
10961
10962
10963
10964
10965
10966
10967
10968
10969
10970
10971
10972
10973
10974
10975
10976
10977
10978
10979
10980
10981
10982
10983
10984
10985
10986
10987
10988
10989
10990
10991
10992
10993
10994
10995
10996
10997
10998
10999
11000
11001
11002
11003
11004
11005
11006
11007
11008
11009
11010
11011
11012
11013
11014
11015
11016
11017
11018
11019
11020
11021
11022
11023
11024
11025
11026
11027
11028
11029
11030
11031
11032
11033
11034
11035
11036
11037
11038
11039
11040
11041
11042
11043
11044
11045
11046
11047
11048
11049
11050
11051
11052
11053
11054
11055
11056
11057
11058
11059
11060
11061
11062
11063
11064
11065
11066
11067
11068
11069
11070
11071
11072
11073
11074
11075
11076
11077
11078
11079
11080
11081
11082
11083
11084
11085
11086
11087
11088
11089
11090
11091
11092
11093
11094
11095
11096
11097
11098
11099
11100
11101
11102
11103
11104
11105
11106
11107
11108
11109
11110
11111
11112
11113
11114
11115
11116
11117
11118
11119
11120
11121
11122
11123
11124
11125
11126
11127
11128
11129
11130
11131
11132
11133
11134
11135
11136
11137
11138
11139
11140
11141
11142
11143
11144
11145
11146
11147
11148
11149
11150
11151
11152
11153
11154
11155
11156
11157
11158
11159
11160
11161
11162
11163
11164
11165
11166
11167
11168
11169
11170
11171
11172
11173
11174
11175
11176
11177
11178
11179
11180
11181
11182
11183
11184
11185
11186
11187
11188
11189
11190
11191
11192
11193
11194
11195
11196
11197
11198
11199
11200
11201
11202
11203
11204
11205
11206
11207
11208
11209
11210
11211
11212
11213
11214
11215
11216
11217
11218
11219
11220
11221
11222
11223
11224
11225
11226
11227
11228
11229
11230
11231
11232
11233
11234
11235
11236
11237
11238
11239
11240
11241
11242
11243
11244
11245
11246
11247
11248
11249
11250
11251
11252
11253
11254
11255
11256
11257
11258
11259
11260
11261
11262
11263
11264
11265
11266
11267
11268
11269
11270
11271
11272
11273
11274
11275
11276
11277
11278
11279
11280
11281
11282
11283
11284
11285
11286
11287
11288
11289
11290
11291
11292
11293
11294
11295
11296
11297
11298
11299
11300
11301
11302
11303
11304
11305
11306
11307
11308
11309
11310
11311
11312
11313
11314
11315
11316
11317
11318
11319
11320
11321
11322
11323
11324
11325
11326
11327
11328
11329
11330
11331
11332
11333
11334
11335
11336
11337
11338
11339
11340
11341
11342
11343
11344
11345
11346
11347
11348
11349
11350
11351
11352
11353
11354
11355
11356
11357
11358
11359
11360
11361
11362
11363
11364
11365
11366
11367
11368
11369
11370
11371
11372
11373
11374
11375
11376
11377
11378
11379
11380
11381
11382
11383
11384
11385
11386
11387
11388
11389
11390
11391
11392
11393
11394
11395
11396
11397
11398
11399
11400
11401
11402
11403
11404
11405
11406
11407
11408
11409
11410
11411
11412
11413
11414
11415
11416
11417
11418
11419
11420
11421
11422
11423
11424
11425
11426
11427
11428
11429
11430
11431
11432
11433
11434
11435
11436
11437
11438
11439
11440
11441
11442
11443
11444
11445
11446
11447
11448
11449
11450
11451
11452
11453
11454
11455
11456
11457
11458
11459
11460
11461
11462
11463
11464
11465
11466
11467
11468
11469
11470
11471
11472
11473
11474
11475
11476
11477
11478
11479
11480
11481
11482
11483
11484
11485
11486
11487
11488
11489
11490
11491
11492
11493
11494
11495
11496
11497
11498
11499
11500
11501
11502
11503
11504
11505
11506
11507
11508
11509
11510
11511
11512
11513
11514
11515
11516
11517
11518
11519
11520
11521
11522
11523
11524
11525
11526
11527
11528
11529
11530
11531
11532
11533
11534
11535
11536
11537
11538
11539
11540
11541
11542
11543
11544
11545
11546
11547
11548
11549
11550
11551
11552
11553
11554
11555
11556
11557
11558
11559
11560
11561
11562
11563
11564
11565
11566
11567
11568
11569
11570
11571
11572
11573
11574
11575
11576
11577
11578
11579
11580
11581
11582
11583
11584
11585
11586
11587
11588
11589
11590
11591
11592
11593
11594
11595
11596
11597
11598
11599
11600
11601
11602
11603
11604
11605
11606
11607
11608
11609
11610
11611
11612
11613
11614
11615
11616
11617
11618
11619
11620
11621
11622
11623
11624
11625
11626
11627
11628
11629
11630
11631
11632
11633
11634
11635
11636
11637
11638
11639
11640
11641
11642
11643
11644
11645
11646
11647
11648
11649
11650
11651
11652
11653
11654
11655
11656
11657
11658
11659
11660
11661
11662
11663
11664
11665
11666
11667
11668
11669
11670
11671
11672
11673
11674
11675
11676
11677
11678
11679
11680
11681
11682
11683
11684
11685
11686
11687
11688
11689
11690
11691
11692
11693
11694
11695
11696
11697
11698
11699
11700
11701
11702
11703
11704
11705
11706
11707
11708
11709
11710
11711
11712
11713
11714
11715
11716
11717
11718
11719
11720
11721
11722
11723
11724
11725
11726
11727
11728
11729
11730
11731
11732
11733
11734
11735
11736
11737
11738
11739
11740
11741
11742
11743
11744
11745
11746
11747
11748
11749
11750
11751
11752
11753
11754
11755
11756
11757
11758
11759
11760
11761
11762
11763
11764
11765
11766
11767
11768
11769
11770
11771
11772
11773
11774
11775
11776
11777
11778
11779
11780
11781
11782
11783
11784
11785
11786
11787
11788
11789
11790
11791
11792
11793
11794
11795
11796
11797
11798
11799
11800
11801
11802
11803
11804
11805
11806
11807
11808
11809
11810
11811
11812
11813
11814
11815
11816
11817
11818
11819
11820
11821
11822
11823
11824
11825
11826
11827
11828
11829
11830
11831
11832
11833
11834
11835
11836
11837
11838
11839
11840
11841
11842
11843
11844
11845
11846
11847
11848
11849
11850
11851
11852
11853
11854
11855
11856
11857
11858
11859
11860
11861
11862
11863
11864
11865
11866
11867
11868
11869
11870
11871
11872
11873
11874
11875
11876
11877
11878
11879
11880
11881
11882
11883
11884
11885
11886
11887
11888
11889
11890
11891
11892
11893
11894
11895
11896
11897
11898
11899
11900
11901
11902
11903
11904
11905
11906
11907
11908
11909
11910
11911
11912
11913
11914
11915
11916
11917
11918
11919
11920
11921
11922
11923
11924
11925
11926
11927
11928
11929
11930
11931
11932
11933
11934
11935
11936
11937
11938
11939
11940
11941
11942
11943
11944
11945
11946
11947
11948
11949
11950
11951
11952
11953
11954
11955
11956
11957
11958
11959
11960
11961
11962
11963
11964
11965
11966
11967
11968
11969
11970
11971
11972
11973
11974
11975
11976
11977
11978
11979
11980
11981
11982
11983
11984
11985
11986
11987
11988
11989
11990
11991
11992
11993
11994
11995
11996
11997
11998
11999
12000
12001
12002
12003
12004
12005
12006
12007
12008
12009
12010
12011
12012
12013
12014
12015
12016
12017
12018
12019
12020
12021
12022
12023
12024
12025
12026
12027
12028
12029
12030
12031
12032
12033
12034
12035
12036
12037
12038
12039
12040
12041
12042
12043
12044
12045
12046
12047
12048
12049
12050
12051
12052
12053
12054
12055
12056
12057
12058
12059
12060
12061
12062
12063
12064
12065
12066
12067
12068
12069
12070
12071
12072
12073
12074
12075
12076
12077
12078
12079
12080
12081
12082
12083
12084
12085
12086
12087
12088
12089
12090
12091
12092
12093
12094
12095
12096
12097
12098
12099
12100
12101
12102
12103
12104
12105
12106
12107
12108
12109
12110
12111
12112
12113
12114
12115
12116
12117
12118
12119
12120
12121
12122
12123
12124
12125
12126
12127
12128
12129
12130
12131
12132
12133
12134
12135
12136
12137
12138
12139
12140
12141
12142
12143
12144
12145
12146
12147
12148
12149
12150
12151
12152
12153
12154
12155
12156
12157
12158
12159
12160
12161
12162
12163
12164
12165
12166
12167
12168
12169
12170
12171
12172
12173
12174
12175
12176
12177
12178
12179
12180
12181
12182
12183
12184
12185
12186
12187
12188
12189
12190
12191
12192
12193
12194
12195
12196
12197
12198
12199
12200
12201
12202
12203
12204
12205
12206
12207
12208
12209
12210
12211
12212
12213
12214
12215
12216
12217
12218
12219
12220
12221
12222
12223
12224
12225
12226
12227
12228
12229
12230
12231
12232
12233
12234
12235
12236
12237
12238
12239
12240
12241
12242
12243
12244
12245
12246
12247
12248
12249
12250
12251
12252
12253
12254
12255
12256
12257
12258
12259
12260
12261
12262
12263
12264
12265
12266
12267
12268
12269
12270
12271
12272
12273
12274
12275
12276
12277
12278
12279
12280
12281
12282
12283
12284
12285
12286
12287
12288
12289
12290
12291
12292
12293
12294
12295
12296
12297
12298
12299
12300
12301
12302
12303
12304
12305
12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
12325
12326
12327
12328
12329
12330
12331
12332
12333
12334
12335
12336
12337
12338
12339
12340
12341
12342
12343
12344
12345
12346
12347
12348
12349
12350
12351
12352
12353
12354
12355
12356
12357
12358
12359
12360
12361
12362
12363
12364
12365
12366
12367
12368
12369
12370
12371
12372
12373
12374
12375
12376
12377
12378
12379
12380
12381
12382
12383
12384
12385
12386
12387
12388
12389
12390
12391
12392
12393
12394
12395
12396
12397
12398
12399
12400
12401
12402
12403
12404
12405
12406
12407
12408
12409
12410
12411
12412
12413
12414
12415
12416
12417
12418
12419
12420
12421
12422
12423
12424
12425
12426
12427
12428
12429
12430
12431
12432
12433
12434
12435
12436
12437
12438
12439
12440
12441
12442
12443
12444
12445
12446
12447
12448
12449
12450
12451
12452
12453
12454
12455
12456
12457
12458
12459
12460
12461
12462
12463
12464
12465
12466
12467
12468
12469
12470
12471
12472
12473
12474
12475
12476
12477
12478
12479
12480
12481
12482
12483
12484
12485
12486
12487
12488
12489
12490
12491
12492
12493
12494
12495
12496
12497
12498
12499
12500
12501
12502
12503
12504
12505
12506
12507
12508
12509
12510
12511
12512
12513
12514
12515
12516
12517
12518
12519
12520
12521
12522
12523
12524
12525
12526
12527
12528
12529
12530
12531
12532
12533
12534
12535
12536
12537
12538
12539
12540
12541
12542
12543
12544
12545
12546
12547
12548
12549
12550
12551
12552
12553
12554
12555
12556
12557
12558
12559
12560
12561
12562
12563
12564
12565
12566
12567
12568
12569
12570
12571
12572
12573
12574
12575
12576
12577
12578
12579
12580
12581
12582
12583
12584
12585
12586
12587
12588
12589
12590
12591
12592
12593
12594
12595
12596
12597
12598
12599
12600
12601
12602
12603
12604
12605
12606
12607
12608
12609
12610
12611
12612
12613
12614
12615
12616
12617
12618
12619
12620
12621
12622
12623
12624
12625
12626
12627
12628
12629
12630
12631
12632
12633
12634
12635
12636
12637
12638
12639
12640
12641
12642
12643
12644
12645
12646
12647
12648
12649
12650
12651
12652
12653
12654
12655
12656
12657
12658
12659
12660
12661
12662
12663
12664
12665
12666
12667
12668
12669
12670
12671
12672
12673
12674
12675
12676
12677
12678
12679
12680
12681
12682
12683
12684
12685
12686
12687
12688
12689
12690
12691
12692
12693
12694
12695
12696
12697
12698
12699
12700
12701
12702
12703
12704
12705
12706
12707
12708
12709
12710
12711
12712
12713
12714
12715
12716
12717
12718
12719
12720
12721
12722
12723
12724
12725
12726
12727
12728
12729
12730
12731
12732
12733
12734
12735
12736
12737
12738
12739
12740
12741
12742
12743
12744
12745
12746
12747
12748
12749
12750
12751
12752
12753
12754
12755
12756
12757
12758
12759
12760
12761
12762
12763
12764
12765
12766
12767
12768
12769
12770
12771
12772
12773
12774
12775
12776
12777
12778
12779
12780
12781
12782
12783
12784
12785
12786
12787
12788
12789
12790
12791
12792
12793
12794
12795
12796
12797
12798
12799
12800
12801
12802
12803
12804
12805
12806
12807
12808
12809
12810
12811
12812
12813
12814
12815
12816
12817
12818
12819
12820
12821
12822
12823
12824
12825
12826
12827
12828
12829
12830
12831
12832
12833
12834
12835
12836
12837
12838
12839
12840
12841
12842
12843
12844
12845
12846
12847
12848
12849
12850
12851
12852
12853
12854
12855
12856
12857
12858
12859
12860
12861
12862
12863
12864
12865
12866
12867
12868
12869
12870
12871
12872
12873
12874
12875
12876
12877
12878
12879
12880
12881
12882
12883
12884
12885
12886
12887
12888
12889
12890
12891
12892
12893
12894
12895
12896
12897
12898
12899
12900
12901
12902
12903
12904
12905
12906
12907
12908
12909
12910
12911
12912
12913
12914
12915
12916
12917
12918
12919
12920
12921
12922
12923
12924
12925
12926
12927
12928
12929
12930
12931
12932
12933
12934
12935
12936
12937
12938
12939
12940
12941
12942
12943
12944
12945
12946
12947
12948
12949
12950
12951
12952
12953
12954
12955
12956
12957
12958
12959
12960
12961
12962
12963
12964
12965
12966
12967
12968
12969
12970
12971
12972
12973
12974
12975
12976
12977
12978
12979
12980
12981
12982
12983
12984
12985
12986
12987
12988
12989
12990
12991
12992
12993
12994
12995
12996
12997
12998
12999
13000
13001
13002
13003
13004
13005
13006
13007
13008
13009
13010
13011
13012
13013
13014
13015
13016
13017
13018
13019
13020
13021
13022
13023
13024
13025
13026
13027
13028
13029
13030
13031
13032
13033
13034
13035
13036
13037
13038
13039
13040
13041
13042
13043
13044
13045
13046
13047
13048
13049
13050
13051
13052
13053
13054
13055
13056
13057
13058
13059
13060
13061
13062
13063
13064
13065
13066
13067
13068
13069
13070
13071
13072
13073
13074
13075
13076
13077
13078
13079
13080
13081
13082
13083
13084
13085
13086
13087
13088
13089
13090
13091
13092
13093
13094
13095
13096
13097
13098
13099
13100
13101
13102
13103
13104
13105
13106
13107
13108
13109
13110
13111
13112
13113
13114
13115
13116
13117
13118
13119
13120
13121
13122
13123
13124
13125
13126
13127
13128
13129
13130
13131
13132
13133
13134
13135
13136
13137
13138
13139
13140
13141
13142
13143
13144
13145
13146
13147
13148
13149
13150
13151
13152
13153
13154
13155
13156
13157
13158
13159
13160
13161
13162
13163
13164
13165
13166
13167
13168
13169
13170
13171
13172
13173
13174
13175
13176
13177
13178
13179
13180
13181
13182
13183
13184
13185
13186
13187
13188
13189
13190
13191
13192
13193
13194
13195
13196
13197
13198
13199
13200
13201
13202
13203
13204
13205
13206
13207
13208
13209
13210
13211
13212
13213
13214
13215
13216
13217
13218
13219
13220
13221
13222
13223
13224
13225
13226
13227
13228
13229
13230
13231
13232
13233
13234
13235
13236
13237
13238
13239
13240
13241
13242
13243
13244
13245
13246
13247
13248
13249
13250
13251
13252
13253
13254
13255
13256
13257
13258
13259
13260
13261
13262
13263
13264
13265
13266
13267
13268
13269
13270
13271
13272
13273
13274
13275
13276
13277
13278
13279
13280
13281
13282
13283
13284
13285
13286
13287
13288
13289
13290
13291
13292
13293
13294
13295
13296
13297
13298
13299
13300
13301
13302
13303
13304
13305
13306
13307
13308
13309
13310
13311
13312
13313
13314
13315
13316
13317
13318
13319
13320
13321
13322
13323
13324
13325
13326
13327
13328
13329
13330
13331
13332
13333
13334
13335
13336
13337
13338
13339
13340
13341
13342
13343
13344
13345
13346
13347
13348
13349
13350
13351
13352
13353
13354
13355
13356
13357
13358
13359
13360
13361
13362
13363
13364
13365
13366
13367
13368
13369
13370
13371
13372
13373
13374
13375
13376
13377
13378
13379
13380
13381
13382
13383
13384
13385
13386
13387
13388
13389
13390
13391
13392
13393
13394
13395
13396
13397
13398
13399
13400
13401
13402
13403
13404
13405
13406
13407
13408
13409
13410
13411
13412
13413
13414
13415
13416
13417
13418
13419
13420
13421
13422
13423
13424
13425
13426
13427
13428
13429
13430
13431
13432
13433
13434
13435
13436
13437
13438
13439
13440
13441
13442
13443
13444
13445
13446
13447
13448
13449
13450
13451
13452
13453
13454
13455
13456
13457
13458
13459
13460
13461
13462
13463
13464
13465
13466
13467
13468
13469
13470
13471
13472
13473
13474
13475
13476
13477
13478
13479
13480
13481
13482
13483
13484
13485
13486
13487
13488
13489
13490
13491
13492
13493
13494
13495
13496
13497
13498
13499
13500
13501
13502
13503
13504
13505
13506
13507
13508
13509
13510
13511
13512
13513
13514
13515
13516
13517
13518
13519
13520
13521
13522
13523
13524
13525
13526
13527
13528
13529
13530
13531
13532
13533
13534
13535
13536
13537
13538
13539
13540
13541
13542
13543
13544
13545
13546
13547
13548
13549
13550
13551
13552
13553
13554
13555
13556
13557
13558
13559
13560
13561
13562
13563
13564
13565
13566
13567
13568
13569
13570
13571
13572
13573
13574
13575
13576
13577
13578
13579
13580
13581
13582
13583
13584
13585
13586
13587
13588
13589
13590
13591
13592
13593
13594
13595
13596
13597
13598
13599
13600
13601
13602
13603
13604
13605
13606
13607
13608
13609
13610
13611
13612
13613
13614
13615
13616
13617
13618
13619
13620
13621
13622
13623
13624
13625
13626
13627
13628
13629
13630
13631
13632
13633
13634
13635
13636
13637
13638
13639
13640
13641
13642
13643
13644
13645
13646
13647
13648
13649
13650
13651
13652
13653
13654
13655
13656
13657
13658
13659
13660
13661
13662
13663
13664
13665
13666
13667
13668
13669
13670
13671
13672
13673
13674
13675
13676
13677
13678
13679
13680
13681
13682
13683
13684
13685
13686
13687
13688
13689
13690
13691
13692
13693
13694
13695
13696
13697
13698
13699
13700
13701
13702
13703
13704
13705
13706
13707
13708
13709
13710
13711
13712
13713
13714
13715
13716
13717
13718
13719
13720
13721
13722
13723
13724
13725
13726
13727
13728
13729
13730
13731
13732
13733
13734
13735
13736
13737
13738
13739
13740
13741
13742
13743
13744
13745
13746
13747
13748
13749
13750
13751
13752
13753
13754
13755
13756
13757
13758
13759
13760
13761
13762
13763
13764
13765
13766
13767
13768
13769
13770
13771
13772
13773
13774
13775
13776
13777
13778
13779
13780
13781
13782
13783
13784
13785
13786
13787
13788
13789
13790
13791
13792
13793
13794
13795
13796
13797
13798
13799
13800
13801
13802
13803
13804
13805
13806
13807
13808
13809
13810
13811
13812
13813
13814
13815
13816
13817
13818
13819
13820
13821
13822
13823
13824
13825
13826
13827
13828
13829
13830
13831
13832
13833
13834
13835
13836
13837
13838
13839
13840
13841
13842
13843
13844
13845
13846
13847
13848
13849
13850
13851
13852
13853
13854
13855
13856
13857
13858
13859
13860
13861
13862
13863
13864
13865
13866
13867
13868
13869
13870
13871
13872
13873
13874
13875
13876
13877
13878
13879
13880
13881
13882
13883
13884
13885
13886
13887
13888
13889
13890
13891
13892
13893
13894
13895
13896
13897
13898
13899
13900
13901
13902
13903
13904
13905
13906
13907
13908
13909
13910
13911
13912
13913
13914
13915
13916
13917
13918
13919
13920
13921
13922
13923
13924
13925
13926
13927
13928
13929
13930
13931
13932
13933
13934
13935
13936
13937
13938
13939
13940
13941
13942
13943
13944
13945
13946
13947
13948
13949
13950
13951
13952
13953
13954
13955
13956
13957
13958
13959
13960
13961
13962
13963
13964
13965
13966
13967
13968
13969
13970
13971
13972
13973
13974
13975
13976
13977
13978
13979
13980
13981
13982
13983
13984
13985
13986
13987
13988
13989
13990
13991
13992
13993
13994
13995
13996
13997
13998
13999
14000
14001
14002
14003
14004
14005
14006
14007
14008
14009
14010
14011
14012
14013
14014
14015
14016
14017
14018
14019
14020
14021
14022
14023
14024
14025
14026
14027
14028
14029
14030
14031
14032
14033
14034
14035
14036
14037
14038
14039
14040
14041
14042
14043
14044
14045
14046
14047
14048
14049
14050
14051
14052
14053
14054
14055
14056
14057
14058
14059
14060
14061
14062
14063
14064
14065
14066
14067
14068
14069
14070
14071
14072
14073
14074
14075
14076
14077
14078
14079
14080
14081
14082
14083
14084
14085
14086
14087
14088
14089
14090
14091
14092
14093
14094
14095
14096
14097
14098
14099
14100
14101
14102
14103
14104
14105
14106
14107
14108
14109
14110
14111
14112
14113
14114
14115
14116
14117
14118
14119
14120
14121
14122
14123
14124
14125
14126
14127
14128
14129
14130
14131
14132
14133
14134
14135
14136
14137
14138
14139
14140
14141
14142
14143
14144
14145
14146
14147
14148
14149
14150
14151
14152
14153
14154
14155
14156
14157
14158
14159
14160
14161
14162
14163
14164
14165
14166
14167
14168
14169
14170
14171
14172
14173
14174
14175
14176
14177
14178
14179
14180
14181
14182
14183
14184
14185
14186
14187
14188
14189
14190
14191
14192
14193
14194
14195
14196
14197
14198
14199
14200
14201
14202
14203
14204
14205
14206
14207
14208
14209
14210
14211
14212
14213
14214
14215
14216
14217
14218
14219
14220
14221
14222
14223
14224
14225
14226
14227
14228
14229
14230
14231
14232
14233
14234
14235
14236
14237
14238
14239
14240
14241
14242
14243
14244
14245
14246
14247
14248
14249
14250
14251
14252
14253
14254
14255
14256
14257
14258
14259
14260
14261
14262
14263
14264
14265
14266
14267
14268
14269
14270
14271
14272
14273
14274
14275
14276
14277
14278
14279
14280
14281
14282
14283
14284
14285
14286
14287
14288
14289
14290
14291
14292
14293
14294
14295
14296
14297
14298
14299
14300
14301
14302
14303
14304
14305
14306
14307
14308
14309
14310
14311
14312
14313
14314
14315
14316
14317
14318
14319
14320
14321
14322
14323
14324
14325
14326
14327
14328
14329
14330
14331
14332
14333
14334
14335
14336
14337
14338
14339
14340
14341
14342
14343
14344
14345
14346
14347
14348
14349
14350
14351
14352
14353
14354
14355
14356
14357
14358
14359
14360
14361
14362
14363
14364
14365
14366
14367
14368
14369
14370
14371
14372
14373
14374
14375
14376
14377
14378
14379
14380
14381
14382
14383
14384
14385
14386
14387
14388
14389
14390
14391
14392
14393
14394
14395
14396
14397
14398
14399
14400
14401
14402
14403
14404
14405
14406
14407
14408
14409
14410
14411
14412
14413
14414
14415
14416
14417
14418
14419
14420
14421
14422
14423
14424
14425
14426
14427
14428
14429
14430
14431
14432
14433
14434
14435
14436
14437
14438
14439
14440
14441
14442
14443
14444
14445
14446
14447
14448
14449
14450
14451
14452
14453
14454
14455
14456
14457
14458
14459
14460
14461
14462
14463
14464
14465
14466
14467
14468
14469
14470
14471
14472
14473
14474
14475
14476
14477
14478
14479
14480
14481
14482
14483
14484
14485
14486
14487
14488
14489
14490
14491
14492
14493
14494
14495
14496
14497
14498
14499
14500
14501
14502
14503
14504
14505
14506
14507
14508
14509
14510
14511
14512
14513
14514
14515
14516
14517
14518
14519
14520
14521
14522
14523
14524
14525
14526
14527
14528
14529
14530
14531
14532
14533
14534
14535
14536
14537
14538
14539
14540
14541
14542
14543
14544
14545
14546
14547
14548
14549
14550
14551
14552
14553
14554
14555
14556
14557
14558
14559
14560
14561
14562
14563
14564
14565
14566
14567
14568
14569
14570
14571
14572
14573
14574
14575
14576
14577
14578
14579
14580
14581
14582
14583
14584
14585
14586
14587
14588
14589
14590
14591
14592
14593
14594
14595
14596
14597
14598
14599
14600
14601
14602
14603
14604
14605
14606
14607
14608
14609
14610
14611
14612
14613
14614
14615
14616
14617
14618
14619
14620
14621
14622
14623
14624
14625
14626
14627
14628
14629
14630
14631
14632
14633
14634
14635
14636
14637
14638
14639
14640
14641
14642
14643
14644
14645
14646
14647
14648
14649
14650
14651
14652
14653
14654
14655
14656
14657
14658
14659
14660
14661
14662
14663
14664
14665
14666
14667
14668
14669
14670
14671
14672
14673
14674
14675
14676
14677
14678
14679
14680
14681
14682
14683
14684
14685
14686
14687
14688
14689
14690
14691
14692
14693
14694
14695
14696
14697
14698
14699
14700
14701
14702
14703
14704
14705
14706
14707
14708
14709
14710
14711
14712
14713
14714
14715
14716
14717
14718
14719
14720
14721
14722
14723
14724
14725
14726
14727
14728
14729
14730
14731
14732
14733
14734
14735
14736
14737
14738
14739
14740
14741
14742
14743
14744
14745
14746
14747
14748
14749
14750
14751
14752
14753
14754
14755
14756
14757
14758
14759
14760
14761
14762
14763
14764
14765
14766
14767
14768
14769
14770
14771
14772
14773
14774
14775
14776
14777
14778
14779
14780
14781
14782
14783
14784
14785
14786
14787
14788
14789
14790
14791
14792
14793
14794
14795
14796
14797
14798
14799
14800
14801
14802
14803
14804
14805
14806
14807
14808
14809
14810
14811
14812
14813
14814
14815
14816
14817
14818
14819
14820
14821
14822
14823
14824
14825
14826
14827
14828
14829
14830
14831
14832
14833
14834
14835
14836
14837
14838
14839
14840
14841
14842
14843
14844
14845
14846
14847
14848
14849
14850
14851
14852
14853
14854
14855
14856
14857
14858
14859
14860
14861
14862
14863
14864
14865
14866
14867
14868
14869
14870
14871
14872
14873
14874
14875
14876
14877
14878
14879
14880
14881
14882
14883
14884
14885
14886
14887
14888
14889
14890
14891
14892
14893
14894
14895
14896
14897
14898
14899
14900
14901
14902
14903
14904
14905
14906
14907
14908
14909
14910
14911
14912
14913
14914
14915
14916
14917
14918
14919
14920
14921
14922
14923
14924
14925
14926
14927
14928
14929
14930
14931
14932
14933
14934
14935
14936
14937
14938
14939
14940
14941
14942
14943
14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
14969
14970
14971
14972
14973
14974
14975
14976
14977
14978
14979
14980
14981
14982
14983
14984
14985
14986
14987
14988
14989
14990
14991
14992
14993
14994
14995
14996
14997
14998
14999
15000
15001
15002
15003
15004
15005
15006
15007
15008
15009
15010
15011
15012
15013
15014
15015
15016
15017
15018
15019
15020
15021
15022
15023
15024
15025
15026
15027
15028
15029
15030
15031
15032
15033
15034
15035
15036
15037
15038
15039
15040
15041
15042
15043
15044
15045
15046
15047
15048
15049
15050
15051
15052
15053
15054
15055
15056
15057
15058
15059
15060
15061
15062
15063
15064
15065
15066
15067
15068
15069
15070
15071
15072
15073
15074
15075
15076
15077
15078
15079
15080
15081
15082
15083
15084
15085
15086
15087
15088
15089
15090
15091
15092
15093
15094
15095
15096
15097
15098
15099
15100
15101
15102
15103
15104
15105
15106
15107
15108
15109
15110
15111
15112
15113
15114
15115
15116
15117
15118
15119
15120
15121
15122
15123
15124
15125
15126
15127
15128
15129
15130
15131
15132
15133
15134
15135
15136
15137
15138
15139
15140
15141
15142
15143
15144
15145
15146
15147
15148
15149
15150
15151
15152
15153
15154
15155
15156
15157
15158
15159
15160
15161
15162
15163
15164
15165
15166
15167
15168
15169
15170
15171
15172
15173
15174
15175
15176
15177
15178
15179
15180
15181
15182
15183
15184
15185
15186
15187
15188
15189
15190
15191
15192
15193
15194
15195
15196
15197
15198
15199
15200
15201
15202
15203
15204
15205
15206
15207
15208
15209
15210
15211
15212
15213
15214
15215
15216
15217
15218
15219
15220
15221
15222
15223
15224
15225
15226
15227
15228
15229
15230
15231
15232
15233
15234
15235
15236
15237
15238
15239
15240
15241
15242
15243
15244
15245
15246
15247
15248
15249
15250
15251
15252
15253
15254
15255
15256
15257
15258
15259
15260
15261
15262
15263
15264
15265
15266
15267
15268
15269
15270
15271
15272
15273
15274
15275
15276
15277
15278
15279
15280
15281
15282
15283
15284
15285
15286
15287
15288
15289
15290
15291
15292
15293
15294
15295
15296
15297
15298
15299
15300
15301
15302
15303
15304
15305
15306
15307
15308
15309
15310
15311
15312
15313
15314
15315
15316
15317
15318
15319
15320
15321
15322
15323
15324
15325
15326
15327
15328
15329
15330
15331
15332
15333
15334
15335
15336
15337
15338
15339
15340
15341
15342
15343
15344
15345
15346
15347
15348
15349
15350
15351
15352
15353
15354
15355
15356
15357
15358
15359
15360
15361
15362
15363
15364
15365
15366
15367
15368
15369
15370
15371
15372
15373
15374
15375
15376
15377
15378
15379
15380
15381
15382
15383
15384
15385
15386
15387
15388
15389
15390
15391
15392
15393
15394
15395
15396
15397
15398
15399
15400
15401
15402
15403
15404
15405
15406
15407
15408
15409
15410
15411
15412
15413
15414
15415
15416
15417
15418
15419
15420
15421
15422
15423
15424
15425
15426
15427
15428
15429
15430
15431
15432
15433
15434
15435
15436
15437
15438
15439
15440
15441
15442
15443
15444
15445
15446
15447
15448
15449
15450
15451
15452
15453
15454
15455
15456
15457
15458
15459
15460
15461
15462
15463
15464
15465
15466
15467
15468
15469
15470
15471
15472
15473
15474
15475
15476
15477
15478
15479
15480
15481
15482
15483
15484
15485
15486
15487
15488
15489
15490
15491
15492
15493
15494
15495
15496
15497
15498
15499
15500
15501
15502
15503
15504
15505
15506
15507
15508
15509
15510
15511
15512
15513
15514
15515
15516
15517
15518
15519
15520
15521
15522
15523
15524
15525
15526
15527
15528
15529
15530
15531
15532
15533
15534
15535
15536
15537
15538
15539
15540
15541
15542
15543
15544
15545
15546
15547
15548
15549
15550
15551
15552
15553
15554
15555
15556
15557
15558
15559
15560
15561
15562
15563
15564
15565
15566
15567
15568
15569
15570
15571
15572
15573
15574
15575
15576
15577
15578
15579
15580
15581
15582
15583
15584
15585
15586
15587
15588
15589
15590
15591
15592
15593
15594
15595
15596
15597
15598
15599
15600
15601
15602
15603
15604
15605
15606
15607
15608
15609
15610
15611
15612
15613
15614
15615
15616
15617
15618
15619
15620
15621
15622
15623
15624
15625
15626
15627
15628
15629
15630
15631
15632
15633
15634
15635
15636
15637
15638
15639
15640
15641
15642
15643
15644
15645
15646
15647
15648
15649
15650
15651
15652
15653
15654
15655
15656
15657
15658
15659
15660
15661
15662
15663
15664
15665
15666
15667
15668
15669
15670
15671
15672
15673
15674
15675
15676
15677
15678
15679
15680
15681
15682
15683
15684
15685
15686
15687
15688
15689
15690
15691
15692
15693
15694
15695
15696
15697
15698
15699
15700
15701
15702
15703
15704
15705
15706
15707
15708
15709
15710
15711
15712
15713
15714
15715
15716
15717
15718
15719
15720
15721
15722
15723
15724
15725
15726
15727
15728
15729
15730
15731
15732
15733
15734
15735
15736
15737
15738
15739
15740
15741
15742
15743
15744
15745
15746
15747
15748
15749
15750
15751
15752
15753
15754
15755
15756
15757
15758
15759
15760
15761
15762
15763
15764
15765
15766
15767
15768
15769
15770
15771
15772
15773
15774
15775
15776
15777
15778
15779
15780
15781
15782
15783
15784
15785
15786
15787
15788
15789
15790
15791
15792
15793
15794
15795
15796
15797
15798
15799
15800
15801
15802
15803
15804
15805
15806
15807
15808
15809
15810
15811
15812
15813
15814
15815
15816
15817
15818
15819
15820
15821
15822
15823
15824
15825
15826
15827
15828
15829
15830
15831
15832
15833
15834
15835
15836
15837
15838
15839
15840
15841
15842
15843
15844
15845
15846
15847
15848
15849
15850
15851
15852
15853
15854
15855
15856
15857
15858
15859
15860
15861
15862
15863
15864
15865
15866
15867
15868
15869
15870
15871
15872
15873
15874
15875
15876
15877
15878
15879
15880
15881
15882
15883
15884
15885
15886
15887
15888
15889
15890
15891
15892
15893
15894
15895
15896
15897
15898
15899
15900
15901
15902
15903
15904
15905
15906
15907
15908
15909
15910
15911
15912
15913
15914
15915
15916
15917
15918
15919
15920
15921
15922
15923
15924
15925
15926
15927
15928
15929
15930
15931
15932
15933
15934
15935
15936
15937
15938
15939
15940
15941
15942
15943
15944
15945
15946
15947
15948
15949
15950
15951
15952
15953
15954
15955
15956
15957
15958
15959
15960
15961
15962
15963
15964
15965
15966
15967
15968
15969
15970
15971
15972
15973
15974
15975
15976
15977
15978
15979
15980
15981
15982
15983
15984
15985
15986
15987
15988
15989
15990
15991
15992
15993
15994
15995
15996
15997
15998
15999
16000
16001
16002
16003
16004
16005
16006
16007
16008
16009
16010
16011
16012
16013
16014
16015
16016
16017
16018
16019
16020
16021
16022
16023
16024
16025
16026
16027
16028
16029
16030
16031
16032
16033
16034
16035
16036
16037
16038
16039
16040
16041
16042
16043
16044
16045
16046
16047
16048
16049
16050
16051
16052
16053
16054
16055
16056
16057
16058
16059
16060
16061
16062
16063
16064
16065
16066
16067
16068
16069
16070
16071
16072
16073
16074
16075
16076
16077
16078
16079
16080
16081
16082
16083
16084
16085
16086
16087
16088
16089
16090
16091
16092
16093
16094
16095
16096
16097
16098
16099
16100
16101
16102
16103
16104
16105
16106
16107
16108
16109
16110
16111
16112
16113
16114
16115
16116
16117
16118
16119
16120
16121
16122
16123
16124
16125
16126
16127
16128
16129
16130
16131
16132
16133
16134
16135
16136
16137
16138
16139
16140
16141
16142
16143
16144
16145
16146
16147
16148
16149
16150
16151
16152
16153
16154
16155
16156
16157
16158
16159
16160
16161
16162
16163
16164
16165
16166
16167
16168
16169
16170
16171
16172
16173
16174
16175
16176
16177
16178
16179
16180
16181
16182
16183
16184
16185
16186
16187
16188
16189
16190
16191
16192
16193
16194
16195
16196
16197
16198
16199
16200
16201
16202
16203
16204
16205
16206
16207
16208
16209
16210
16211
16212
16213
16214
16215
16216
16217
16218
16219
16220
16221
16222
16223
16224
16225
16226
16227
16228
16229
16230
16231
16232
16233
16234
16235
16236
16237
16238
16239
16240
16241
16242
16243
16244
16245
16246
16247
16248
16249
16250
16251
16252
16253
16254
16255
16256
16257
16258
16259
16260
16261
16262
16263
16264
16265
16266
16267
16268
16269
16270
16271
16272
16273
16274
16275
16276
16277
16278
16279
16280
16281
16282
16283
16284
16285
16286
16287
16288
16289
16290
16291
16292
16293
16294
16295
16296
16297
16298
16299
16300
16301
16302
16303
16304
16305
16306
16307
16308
16309
16310
16311
16312
16313
16314
16315
16316
16317
16318
16319
16320
16321
16322
16323
16324
16325
16326
16327
16328
16329
16330
16331
16332
16333
16334
16335
16336
16337
16338
16339
16340
16341
16342
16343
16344
16345
16346
16347
16348
16349
16350
16351
16352
16353
16354
16355
16356
16357
16358
16359
16360
16361
16362
16363
16364
16365
16366
16367
16368
16369
16370
16371
16372
16373
16374
16375
16376
16377
16378
16379
16380
16381
16382
16383
16384
16385
16386
16387
16388
16389
16390
16391
16392
16393
16394
16395
16396
16397
16398
16399
16400
16401
16402
16403
16404
16405
16406
16407
16408
16409
16410
16411
16412
16413
16414
16415
16416
16417
16418
16419
16420
16421
16422
16423
16424
16425
16426
16427
16428
16429
16430
16431
16432
16433
16434
16435
16436
16437
16438
16439
16440
16441
16442
16443
16444
16445
16446
16447
16448
16449
16450
16451
16452
16453
16454
16455
16456
16457
16458
16459
16460
16461
16462
16463
16464
16465
16466
16467
16468
16469
16470
16471
16472
16473
16474
16475
16476
16477
16478
16479
16480
16481
16482
16483
16484
16485
16486
16487
16488
16489
16490
16491
16492
16493
16494
16495
16496
16497
16498
16499
16500
16501
16502
16503
16504
16505
16506
16507
16508
16509
16510
16511
16512
16513
16514
16515
16516
16517
16518
16519
16520
16521
16522
16523
16524
16525
16526
16527
16528
16529
16530
16531
16532
16533
16534
16535
16536
16537
16538
16539
16540
16541
16542
16543
16544
16545
16546
16547
16548
16549
16550
16551
16552
16553
16554
16555
16556
16557
16558
16559
16560
16561
16562
16563
16564
16565
16566
16567
16568
16569
16570
16571
16572
16573
16574
16575
16576
16577
16578
16579
16580
16581
16582
16583
16584
16585
16586
16587
16588
16589
16590
16591
16592
16593
16594
16595
16596
16597
16598
16599
16600
16601
16602
16603
16604
16605
16606
16607
16608
16609
16610
16611
16612
16613
16614
16615
16616
16617
16618
16619
16620
16621
16622
16623
16624
16625
16626
16627
16628
16629
16630
16631
16632
16633
16634
16635
16636
16637
16638
16639
16640
16641
16642
16643
16644
16645
16646
16647
16648
16649
16650
16651
16652
16653
16654
16655
16656
16657
16658
16659
16660
16661
16662
16663
16664
16665
16666
16667
16668
16669
16670
16671
16672
16673
16674
16675
16676
16677
16678
16679
16680
16681
16682
16683
16684
16685
16686
16687
16688
16689
16690
16691
16692
16693
16694
16695
16696
16697
16698
16699
16700
16701
16702
16703
16704
16705
16706
16707
16708
16709
16710
16711
16712
16713
16714
16715
16716
16717
16718
16719
16720
16721
16722
16723
16724
16725
16726
16727
16728
16729
16730
16731
16732
16733
16734
16735
16736
16737
16738
16739
16740
16741
16742
16743
16744
16745
16746
16747
16748
16749
16750
16751
16752
16753
16754
16755
16756
16757
16758
16759
16760
16761
16762
16763
16764
16765
16766
16767
16768
16769
16770
16771
16772
16773
16774
16775
16776
16777
16778
16779
16780
16781
16782
16783
16784
16785
16786
16787
16788
16789
16790
16791
16792
16793
16794
16795
16796
16797
16798
16799
16800
16801
16802
16803
16804
16805
16806
16807
16808
16809
16810
16811
16812
16813
16814
16815
16816
16817
16818
16819
16820
16821
16822
16823
16824
16825
16826
16827
16828
16829
16830
16831
16832
16833
16834
16835
16836
16837
16838
16839
16840
16841
16842
16843
16844
16845
16846
16847
16848
16849
16850
16851
16852
16853
16854
16855
16856
16857
16858
16859
16860
16861
16862
16863
16864
16865
16866
16867
16868
16869
16870
16871
16872
16873
16874
16875
16876
16877
16878
16879
16880
16881
16882
16883
16884
16885
16886
16887
16888
16889
16890
16891
16892
16893
16894
16895
16896
16897
16898
16899
16900
16901
16902
16903
16904
16905
16906
16907
16908
16909
16910
16911
16912
16913
16914
16915
16916
16917
16918
16919
16920
16921
16922
16923
16924
16925
16926
16927
16928
16929
16930
16931
16932
16933
16934
16935
16936
16937
16938
16939
16940
16941
16942
16943
16944
16945
16946
16947
16948
16949
16950
16951
16952
16953
16954
16955
16956
16957
16958
16959
16960
16961
16962
16963
16964
16965
16966
16967
16968
16969
16970
16971
16972
16973
16974
16975
16976
16977
16978
16979
16980
16981
16982
16983
16984
16985
16986
16987
16988
16989
16990
16991
16992
16993
16994
16995
16996
16997
16998
16999
17000
17001
17002
17003
17004
17005
17006
17007
17008
17009
17010
17011
17012
17013
17014
17015
17016
17017
17018
17019
17020
17021
17022
17023
17024
17025
17026
17027
17028
17029
17030
17031
17032
17033
17034
17035
17036
17037
17038
17039
17040
17041
17042
17043
17044
17045
17046
17047
17048
17049
17050
17051
17052
17053
17054
17055
17056
17057
17058
17059
17060
17061
17062
17063
17064
17065
17066
17067
17068
17069
17070
17071
17072
17073
17074
17075
17076
17077
17078
17079
17080
17081
17082
17083
17084
17085
17086
17087
17088
17089
17090
17091
17092
17093
17094
17095
17096
17097
17098
17099
17100
17101
17102
17103
17104
17105
17106
17107
17108
17109
17110
17111
17112
17113
17114
17115
17116
17117
17118
17119
17120
17121
17122
17123
17124
17125
17126
17127
17128
17129
17130
17131
17132
17133
17134
17135
17136
17137
17138
17139
17140
17141
17142
17143
17144
17145
17146
17147
17148
17149
17150
17151
17152
17153
17154
17155
17156
17157
17158
17159
17160
17161
17162
17163
17164
17165
17166
17167
17168
17169
17170
17171
17172
17173
17174
17175
17176
17177
17178
17179
17180
17181
17182
17183
17184
17185
17186
17187
17188
17189
17190
17191
17192
17193
17194
17195
17196
17197
17198
17199
17200
17201
17202
17203
17204
17205
17206
17207
17208
17209
17210
17211
17212
17213
17214
17215
17216
17217
17218
17219
17220
17221
17222
17223
17224
17225
17226
17227
17228
17229
17230
17231
17232
17233
17234
17235
17236
17237
17238
17239
17240
17241
17242
17243
17244
17245
17246
17247
17248
17249
17250
17251
17252
17253
17254
17255
17256
17257
17258
17259
17260
17261
17262
17263
17264
17265
17266
17267
17268
17269
17270
17271
17272
17273
17274
17275
17276
17277
17278
17279
17280
17281
17282
17283
17284
17285
17286
17287
17288
17289
17290
17291
17292
17293
17294
17295
17296
17297
17298
17299
17300
17301
17302
17303
17304
17305
17306
17307
17308
17309
17310
17311
17312
17313
17314
17315
17316
17317
17318
17319
17320
17321
17322
17323
17324
17325
17326
17327
17328
17329
17330
17331
17332
17333
17334
17335
17336
17337
17338
17339
17340
17341
17342
17343
17344
17345
17346
17347
17348
17349
17350
17351
17352
17353
17354
17355
17356
17357
17358
17359
17360
17361
17362
17363
17364
17365
17366
17367
17368
17369
17370
17371
17372
17373
17374
17375
17376
17377
17378
17379
17380
17381
17382
17383
17384
17385
17386
17387
17388
17389
17390
17391
17392
17393
17394
17395
17396
17397
17398
17399
17400
17401
17402
17403
17404
17405
17406
17407
17408
17409
17410
17411
17412
17413
17414
17415
17416
17417
17418
17419
17420
17421
17422
17423
17424
17425
17426
17427
17428
17429
17430
17431
17432
17433
17434
17435
17436
17437
17438
17439
17440
17441
17442
17443
17444
17445
17446
17447
17448
17449
17450
17451
17452
17453
17454
17455
17456
17457
17458
17459
17460
17461
17462
17463
17464
17465
17466
17467
17468
17469
17470
17471
17472
17473
17474
17475
17476
17477
17478
17479
17480
17481
17482
17483
17484
17485
17486
17487
17488
17489
17490
17491
17492
17493
17494
17495
17496
17497
17498
17499
17500
17501
17502
17503
17504
17505
17506
17507
17508
17509
17510
17511
17512
17513
17514
17515
17516
17517
17518
17519
17520
17521
17522
17523
17524
17525
17526
17527
17528
17529
17530
17531
17532
17533
17534
17535
17536
17537
17538
17539
17540
17541
17542
17543
17544
17545
17546
17547
17548
17549
17550
17551
17552
17553
17554
17555
17556
17557
17558
17559
17560
17561
17562
17563
17564
17565
17566
17567
17568
17569
17570
17571
17572
17573
17574
17575
17576
17577
17578
17579
17580
17581
17582
17583
17584
17585
17586
17587
17588
17589
17590
17591
17592
17593
17594
17595
17596
17597
17598
17599
17600
17601
17602
17603
17604
17605
17606
17607
17608
17609
17610
17611
17612
17613
17614
17615
17616
17617
17618
17619
17620
17621
17622
17623
17624
17625
17626
17627
17628
17629
17630
17631
17632
17633
17634
17635
17636
17637
17638
17639
17640
17641
17642
17643
17644
17645
17646
17647
17648
17649
17650
17651
17652
17653
17654
17655
17656
17657
17658
17659
17660
17661
17662
17663
17664
17665
17666
17667
17668
17669
17670
17671
17672
17673
17674
17675
17676
17677
17678
17679
17680
17681
17682
17683
17684
17685
17686
17687
17688
17689
17690
17691
17692
17693
17694
17695
17696
17697
17698
17699
17700
17701
17702
17703
17704
17705
17706
17707
17708
17709
17710
17711
17712
17713
17714
17715
17716
17717
17718
17719
17720
17721
17722
17723
17724
17725
17726
17727
17728
17729
17730
17731
17732
17733
17734
17735
17736
17737
17738
17739
17740
17741
17742
17743
17744
17745
17746
17747
17748
17749
17750
17751
17752
17753
17754
17755
17756
17757
17758
17759
17760
17761
17762
17763
17764
17765
17766
17767
17768
17769
17770
17771
17772
17773
17774
17775
17776
17777
17778
17779
17780
17781
17782
17783
17784
17785
17786
17787
17788
17789
17790
17791
17792
17793
17794
17795
17796
17797
17798
17799
17800
17801
17802
17803
17804
17805
17806
17807
17808
17809
17810
17811
17812
17813
17814
17815
17816
17817
17818
17819
17820
17821
17822
17823
17824
17825
17826
17827
17828
17829
17830
17831
17832
17833
17834
17835
17836
17837
17838
17839
17840
17841
17842
17843
17844
17845
17846
17847
17848
17849
17850
17851
17852
17853
17854
17855
17856
17857
17858
17859
17860
17861
17862
17863
17864
17865
17866
17867
17868
17869
17870
17871
17872
17873
17874
17875
17876
17877
17878
17879
17880
17881
17882
17883
17884
17885
17886
17887
17888
17889
17890
17891
17892
17893
17894
17895
17896
17897
17898
17899
17900
17901
17902
17903
17904
17905
17906
17907
17908
17909
17910
17911
17912
17913
17914
17915
17916
17917
17918
17919
17920
17921
17922
17923
17924
17925
17926
17927
17928
17929
17930
17931
17932
17933
17934
17935
17936
17937
17938
17939
17940
17941
17942
17943
17944
17945
17946
17947
17948
17949
17950
17951
17952
17953
17954
17955
17956
17957
17958
17959
17960
17961
17962
17963
17964
17965
17966
17967
17968
17969
17970
17971
17972
17973
17974
17975
17976
17977
17978
17979
17980
17981
17982
17983
17984
17985
17986
17987
17988
17989
17990
17991
17992
17993
17994
17995
17996
17997
17998
17999
18000
18001
18002
18003
18004
18005
18006
18007
18008
18009
18010
18011
18012
18013
18014
18015
18016
18017
18018
18019
18020
18021
18022
18023
18024
18025
18026
18027
18028
18029
18030
18031
18032
18033
18034
18035
18036
18037
18038
18039
18040
18041
18042
18043
18044
18045
18046
18047
18048
18049
18050
18051
18052
18053
18054
18055
18056
18057
18058
18059
18060
18061
18062
18063
18064
18065
18066
18067
18068
18069
18070
18071
18072
18073
18074
18075
18076
18077
18078
18079
18080
18081
18082
18083
18084
18085
18086
18087
18088
18089
18090
18091
18092
18093
18094
18095
18096
18097
18098
18099
18100
18101
18102
18103
18104
18105
18106
18107
18108
18109
18110
18111
18112
18113
18114
18115
18116
18117
18118
18119
18120
18121
18122
18123
18124
18125
18126
18127
18128
18129
18130
18131
18132
18133
18134
18135
18136
18137
18138
18139
18140
18141
18142
18143
18144
18145
18146
18147
18148
18149
18150
18151
18152
18153
18154
18155
18156
18157
18158
18159
18160
18161
18162
18163
18164
18165
18166
18167
18168
18169
18170
18171
18172
18173
18174
18175
18176
18177
18178
18179
18180
18181
18182
18183
18184
18185
18186
18187
18188
18189
18190
18191
18192
18193
18194
18195
18196
18197
18198
18199
18200
18201
18202
18203
18204
18205
18206
18207
18208
18209
18210
18211
18212
18213
18214
18215
18216
18217
18218
18219
18220
18221
18222
18223
18224
18225
18226
18227
18228
18229
18230
18231
18232
18233
18234
18235
18236
18237
18238
18239
18240
18241
18242
18243
18244
18245
18246
18247
18248
18249
18250
18251
18252
18253
18254
18255
18256
18257
18258
18259
18260
18261
18262
18263
18264
18265
18266
18267
18268
18269
18270
18271
18272
18273
18274
18275
18276
18277
18278
18279
18280
18281
18282
18283
18284
18285
18286
18287
18288
18289
18290
18291
18292
18293
18294
18295
18296
18297
18298
18299
18300
18301
18302
18303
18304
18305
18306
18307
18308
18309
18310
18311
18312
18313
18314
18315
18316
18317
18318
18319
18320
18321
18322
18323
18324
18325
18326
18327
18328
18329
18330
18331
18332
18333
18334
18335
18336
18337
18338
18339
18340
18341
18342
18343
18344
18345
18346
18347
18348
18349
18350
18351
18352
18353
18354
18355
18356
18357
18358
18359
18360
18361
18362
18363
18364
18365
18366
18367
18368
18369
18370
18371
18372
18373
18374
18375
18376
18377
18378
18379
18380
18381
18382
18383
18384
18385
18386
18387
18388
18389
18390
18391
18392
18393
18394
18395
18396
18397
18398
18399
18400
18401
18402
18403
18404
18405
18406
18407
18408
18409
18410
18411
18412
18413
18414
18415
18416
18417
18418
18419
18420
18421
18422
18423
18424
18425
18426
18427
18428
18429
18430
18431
18432
18433
18434
18435
18436
18437
18438
18439
18440
18441
18442
18443
18444
18445
18446
18447
18448
18449
18450
18451
18452
18453
18454
18455
18456
18457
18458
18459
18460
18461
18462
18463
18464
18465
18466
18467
18468
18469
18470
18471
18472
18473
18474
18475
18476
18477
18478
18479
18480
18481
18482
18483
18484
18485
18486
18487
18488
18489
18490
18491
18492
18493
18494
18495
18496
18497
18498
18499
18500
18501
18502
18503
18504
18505
18506
18507
18508
18509
18510
18511
18512
18513
18514
18515
18516
18517
18518
18519
18520
18521
18522
18523
18524
18525
18526
18527
18528
18529
18530
18531
18532
18533
18534
18535
18536
18537
18538
18539
18540
18541
18542
18543
18544
18545
18546
18547
18548
18549
18550
18551
18552
18553
18554
18555
18556
18557
18558
18559
18560
18561
18562
18563
18564
18565
18566
18567
18568
18569
18570
18571
18572
18573
18574
18575
18576
18577
18578
18579
18580
18581
18582
18583
18584
18585
18586
18587
18588
18589
18590
18591
18592
18593
18594
18595
18596
18597
18598
18599
18600
18601
18602
18603
18604
18605
18606
18607
18608
18609
18610
18611
18612
18613
18614
18615
18616
18617
18618
18619
18620
18621
18622
18623
18624
18625
18626
18627
18628
18629
18630
18631
18632
18633
18634
18635
18636
18637
18638
18639
18640
18641
18642
18643
18644
18645
18646
18647
18648
18649
18650
18651
18652
18653
18654
18655
18656
18657
18658
18659
18660
18661
18662
18663
18664
18665
18666
18667
18668
18669
18670
18671
18672
18673
18674
18675
18676
18677
18678
18679
18680
18681
18682
18683
18684
18685
18686
18687
18688
18689
18690
18691
18692
18693
18694
18695
18696
18697
18698
18699
18700
18701
18702
18703
18704
18705
18706
18707
18708
18709
18710
18711
18712
18713
18714
18715
18716
18717
18718
18719
18720
18721
18722
18723
18724
18725
18726
18727
18728
18729
18730
18731
18732
18733
18734
18735
18736
18737
18738
18739
18740
18741
18742
18743
18744
18745
18746
18747
18748
18749
18750
18751
18752
18753
18754
18755
18756
18757
18758
18759
18760
18761
18762
18763
18764
18765
18766
18767
18768
18769
18770
18771
18772
18773
18774
18775
18776
18777
18778
18779
18780
18781
18782
18783
18784
18785
18786
18787
18788
18789
18790
18791
18792
18793
18794
18795
18796
18797
18798
18799
18800
18801
18802
18803
18804
18805
18806
18807
18808
18809
18810
18811
18812
18813
18814
18815
18816
18817
18818
18819
18820
18821
18822
18823
18824
18825
18826
18827
18828
18829
18830
18831
18832
18833
18834
18835
18836
18837
18838
18839
18840
18841
18842
18843
18844
18845
18846
18847
18848
18849
18850
18851
18852
18853
18854
18855
18856
18857
18858
18859
18860
18861
18862
18863
18864
18865
18866
18867
18868
18869
18870
18871
18872
18873
18874
18875
18876
18877
18878
18879
18880
18881
18882
18883
18884
18885
18886
18887
18888
18889
18890
18891
18892
18893
18894
18895
18896
18897
18898
18899
18900
18901
18902
18903
18904
18905
18906
18907
18908
18909
18910
18911
18912
18913
18914
18915
18916
18917
18918
18919
18920
18921
18922
18923
18924
18925
18926
18927
18928
18929
18930
18931
18932
18933
18934
18935
18936
18937
18938
18939
18940
18941
18942
18943
18944
18945
18946
18947
18948
18949
18950
18951
18952
18953
18954
18955
18956
18957
18958
18959
18960
18961
18962
18963
18964
18965
18966
18967
18968
18969
18970
18971
18972
18973
18974
18975
18976
18977
18978
18979
18980
18981
18982
18983
18984
18985
18986
18987
18988
18989
18990
18991
18992
18993
18994
18995
18996
18997
18998
18999
19000
19001
19002
19003
19004
19005
19006
19007
19008
19009
19010
19011
19012
19013
19014
19015
19016
19017
19018
19019
19020
19021
19022
19023
19024
19025
19026
19027
19028
19029
19030
19031
19032
19033
19034
19035
19036
19037
19038
19039
19040
19041
19042
19043
19044
19045
19046
19047
19048
19049
19050
19051
19052
19053
19054
19055
19056
19057
19058
19059
19060
19061
19062
19063
19064
19065
19066
19067
19068
19069
19070
19071
19072
19073
19074
19075
19076
19077
19078
19079
19080
19081
19082
19083
19084
19085
19086
19087
19088
19089
19090
19091
19092
19093
19094
19095
19096
19097
19098
19099
19100
19101
19102
19103
19104
19105
19106
19107
19108
19109
19110
19111
19112
19113
19114
19115
19116
19117
19118
19119
19120
19121
19122
19123
19124
19125
19126
19127
19128
19129
19130
19131
19132
19133
19134
19135
19136
19137
19138
19139
19140
19141
19142
19143
19144
19145
19146
19147
19148
19149
19150
19151
19152
19153
19154
19155
19156
19157
19158
19159
19160
19161
19162
19163
19164
19165
19166
19167
19168
19169
19170
19171
19172
19173
19174
19175
19176
19177
19178
19179
19180
19181
19182
19183
19184
19185
19186
19187
19188
19189
19190
19191
19192
19193
19194
19195
19196
19197
19198
19199
19200
19201
19202
19203
19204
19205
19206
19207
19208
19209
19210
19211
19212
19213
19214
19215
19216
19217
19218
19219
19220
19221
19222
19223
19224
19225
19226
19227
19228
19229
19230
19231
19232
19233
19234
19235
19236
19237
19238
19239
19240
19241
19242
19243
19244
19245
19246
19247
19248
19249
19250
19251
19252
19253
19254
19255
19256
19257
19258
19259
19260
19261
19262
19263
19264
19265
19266
19267
19268
19269
19270
19271
19272
19273
19274
19275
19276
19277
19278
19279
19280
19281
19282
19283
19284
19285
19286
19287
19288
19289
19290
19291
19292
19293
19294
19295
19296
19297
19298
19299
19300
19301
19302
19303
19304
19305
19306
19307
19308
19309
19310
19311
19312
19313
19314
19315
19316
19317
19318
19319
19320
19321
19322
19323
19324
19325
19326
19327
19328
19329
19330
19331
19332
19333
19334
19335
19336
19337
19338
19339
19340
19341
19342
19343
19344
19345
19346
19347
19348
19349
19350
19351
19352
19353
19354
19355
19356
19357
19358
19359
19360
19361
19362
19363
19364
19365
19366
19367
19368
19369
19370
19371
19372
19373
19374
19375
19376
19377
19378
19379
19380
19381
19382
19383
19384
19385
19386
19387
19388
19389
19390
19391
19392
19393
19394
19395
19396
19397
19398
19399
19400
19401
19402
19403
19404
19405
19406
19407
19408
19409
19410
19411
19412
19413
19414
19415
19416
19417
19418
19419
19420
19421
19422
19423
19424
19425
19426
19427
19428
19429
19430
19431
19432
19433
19434
19435
19436
19437
19438
19439
19440
19441
19442
19443
19444
19445
19446
19447
19448
19449
19450
19451
19452
19453
19454
19455
19456
19457
19458
19459
19460
19461
19462
19463
19464
19465
19466
19467
19468
19469
19470
19471
19472
19473
19474
19475
19476
19477
19478
19479
19480
19481
19482
19483
19484
19485
19486
19487
19488
19489
19490
19491
19492
19493
19494
19495
19496
19497
19498
19499
19500
19501
19502
19503
19504
19505
19506
19507
19508
19509
19510
19511
19512
19513
19514
19515
19516
19517
19518
19519
19520
19521
19522
19523
19524
19525
19526
19527
19528
19529
19530
19531
19532
19533
19534
19535
19536
19537
19538
19539
19540
19541
19542
19543
19544
19545
19546
19547
19548
19549
19550
19551
19552
19553
19554
19555
19556
19557
19558
19559
19560
19561
19562
19563
19564
19565
19566
19567
19568
19569
19570
19571
19572
19573
19574
19575
19576
19577
19578
19579
19580
19581
19582
19583
19584
19585
19586
19587
19588
19589
19590
19591
19592
19593
19594
19595
19596
19597
19598
19599
19600
19601
19602
19603
19604
19605
19606
19607
19608
19609
19610
19611
19612
19613
19614
19615
19616
19617
19618
19619
19620
19621
19622
19623
19624
19625
19626
19627
19628
19629
19630
19631
19632
19633
19634
19635
19636
19637
19638
19639
19640
19641
19642
19643
19644
19645
19646
19647
19648
19649
19650
19651
19652
19653
19654
19655
19656
19657
19658
19659
19660
19661
19662
19663
19664
19665
19666
19667
19668
19669
19670
19671
19672
19673
19674
19675
19676
19677
19678
19679
19680
19681
19682
19683
19684
19685
19686
19687
19688
19689
19690
19691
19692
19693
19694
19695
19696
19697
19698
19699
19700
19701
19702
19703
19704
19705
19706
19707
19708
19709
19710
19711
19712
19713
19714
19715
19716
19717
19718
19719
19720
19721
19722
19723
19724
19725
19726
19727
19728
19729
19730
19731
19732
19733
19734
19735
19736
19737
19738
19739
19740
19741
19742
19743
19744
19745
19746
19747
19748
19749
19750
19751
19752
19753
19754
19755
19756
19757
19758
19759
19760
19761
19762
19763
19764
19765
19766
19767
19768
19769
19770
19771
19772
19773
19774
19775
19776
19777
19778
19779
19780
19781
19782
19783
19784
19785
19786
19787
19788
19789
19790
19791
19792
19793
19794
19795
19796
19797
19798
19799
19800
19801
19802
19803
19804
19805
19806
19807
19808
19809
19810
19811
19812
19813
19814
19815
19816
19817
19818
19819
19820
19821
19822
19823
19824
19825
19826
19827
19828
19829
19830
19831
19832
19833
19834
19835
19836
19837
19838
19839
19840
19841
19842
19843
19844
19845
19846
19847
19848
19849
19850
19851
19852
19853
19854
19855
19856
19857
19858
19859
19860
19861
19862
19863
19864
19865
19866
19867
19868
19869
19870
19871
19872
19873
19874
19875
19876
19877
19878
19879
19880
19881
19882
19883
19884
19885
19886
19887
19888
19889
19890
19891
19892
19893
19894
19895
19896
19897
19898
19899
19900
19901
19902
19903
19904
19905
19906
19907
19908
19909
19910
19911
19912
19913
19914
19915
19916
19917
19918
19919
19920
19921
19922
19923
19924
19925
19926
19927
19928
19929
19930
19931
19932
19933
19934
19935
19936
19937
19938
19939
19940
19941
19942
19943
19944
19945
19946
19947
19948
19949
19950
19951
19952
19953
19954
19955
19956
19957
19958
19959
19960
19961
19962
19963
19964
19965
19966
19967
19968
19969
19970
19971
19972
19973
19974
19975
19976
19977
19978
19979
19980
19981
19982
19983
19984
19985
19986
19987
19988
19989
19990
19991
19992
19993
19994
19995
19996
19997
19998
19999
20000
20001
20002
20003
20004
20005
20006
20007
20008
20009
20010
20011
20012
20013
20014
20015
20016
20017
20018
20019
20020
20021
20022
20023
20024
20025
20026
20027
20028
20029
20030
20031
20032
20033
20034
20035
20036
20037
20038
20039
20040
20041
20042
20043
20044
20045
20046
20047
20048
20049
20050
20051
20052
20053
20054
20055
20056
20057
20058
20059
20060
20061
20062
20063
20064
20065
20066
20067
20068
20069
20070
20071
20072
20073
20074
20075
20076
20077
20078
20079
20080
20081
20082
20083
20084
20085
20086
20087
20088
20089
20090
20091
20092
20093
20094
20095
20096
20097
20098
20099
20100
20101
20102
20103
20104
20105
20106
20107
20108
20109
20110
20111
20112
20113
20114
20115
20116
20117
20118
20119
20120
20121
20122
20123
20124
20125
20126
20127
20128
20129
20130
20131
20132
20133
20134
20135
20136
20137
20138
20139
20140
20141
20142
20143
20144
20145
20146
20147
20148
20149
20150
20151
20152
20153
20154
20155
20156
20157
20158
20159
20160
20161
20162
20163
20164
20165
20166
20167
20168
20169
20170
20171
20172
20173
20174
20175
20176
20177
20178
20179
20180
20181
20182
20183
20184
20185
20186
20187
20188
20189
20190
20191
20192
20193
20194
20195
20196
20197
20198
20199
20200
20201
20202
20203
20204
20205
20206
20207
20208
20209
20210
20211
20212
20213
20214
20215
20216
20217
20218
20219
20220
20221
20222
20223
20224
20225
20226
20227
20228
20229
20230
20231
20232
20233
20234
20235
20236
20237
20238
20239
20240
20241
20242
20243
20244
20245
20246
20247
20248
20249
20250
20251
20252
20253
20254
20255
20256
20257
20258
20259
20260
20261
20262
20263
20264
20265
20266
20267
20268
20269
20270
20271
20272
20273
20274
20275
20276
20277
20278
20279
20280
20281
20282
20283
20284
20285
20286
20287
20288
20289
20290
20291
20292
20293
20294
20295
20296
20297
20298
20299
20300
20301
20302
20303
20304
20305
20306
20307
20308
20309
20310
20311
20312
20313
20314
20315
20316
20317
20318
20319
20320
20321
20322
20323
20324
20325
20326
20327
20328
20329
20330
20331
20332
20333
20334
20335
20336
20337
20338
20339
20340
20341
20342
20343
20344
20345
20346
20347
20348
20349
20350
20351
20352
20353
20354
20355
20356
20357
20358
20359
20360
20361
20362
20363
20364
20365
20366
20367
20368
20369
20370
20371
20372
20373
20374
20375
20376
20377
20378
20379
20380
20381
20382
20383
20384
20385
20386
20387
20388
20389
20390
20391
20392
20393
20394
20395
20396
20397
20398
20399
20400
20401
20402
20403
20404
20405
20406
20407
20408
20409
20410
20411
20412
20413
20414
20415
20416
20417
20418
20419
20420
20421
20422
20423
20424
20425
20426
20427
20428
20429
20430
20431
20432
20433
20434
20435
20436
20437
20438
20439
20440
20441
20442
20443
20444
20445
20446
20447
20448
20449
20450
20451
20452
20453
20454
20455
20456
20457
20458
20459
20460
20461
20462
20463
20464
20465
20466
20467
20468
20469
20470
20471
20472
20473
20474
20475
20476
20477
20478
20479
20480
20481
20482
20483
20484
20485
20486
20487
20488
20489
20490
20491
20492
20493
20494
20495
20496
20497
20498
20499
20500
20501
20502
20503
20504
20505
20506
20507
20508
20509
20510
20511
20512
20513
20514
20515
20516
20517
20518
20519
20520
20521
20522
20523
20524
20525
20526
20527
20528
20529
20530
20531
20532
20533
20534
20535
20536
20537
20538
20539
20540
20541
20542
20543
20544
20545
20546
20547
20548
20549
20550
20551
20552
20553
20554
20555
20556
20557
20558
20559
20560
20561
20562
20563
20564
20565
20566
20567
20568
20569
20570
20571
20572
20573
20574
20575
20576
20577
20578
20579
20580
20581
20582
20583
20584
20585
20586
20587
20588
20589
20590
20591
20592
20593
20594
20595
20596
20597
20598
20599
20600
20601
20602
20603
20604
20605
20606
20607
20608
20609
20610
20611
20612
20613
20614
20615
20616
20617
20618
20619
20620
20621
20622
20623
20624
20625
20626
20627
20628
20629
20630
20631
20632
20633
20634
20635
20636
20637
20638
20639
20640
20641
20642
20643
20644
20645
20646
20647
20648
20649
20650
20651
20652
20653
20654
20655
20656
20657
20658
20659
20660
20661
20662
20663
20664
20665
20666
20667
20668
20669
20670
20671
20672
20673
20674
20675
20676
20677
20678
20679
20680
20681
20682
20683
20684
20685
20686
20687
20688
20689
20690
20691
20692
20693
20694
20695
20696
20697
20698
20699
20700
20701
20702
20703
20704
20705
20706
20707
20708
20709
20710
20711
20712
20713
20714
20715
20716
20717
20718
20719
20720
20721
20722
20723
20724
20725
20726
20727
20728
20729
20730
20731
20732
20733
20734
20735
20736
20737
20738
20739
20740
20741
20742
20743
20744
20745
20746
20747
20748
20749
20750
20751
20752
20753
20754
20755
20756
20757
20758
20759
20760
20761
20762
20763
20764
20765
20766
20767
20768
20769
20770
20771
20772
20773
20774
20775
20776
20777
20778
20779
20780
20781
20782
20783
20784
20785
20786
20787
20788
20789
20790
20791
20792
20793
20794
20795
20796
20797
20798
20799
20800
20801
20802
20803
20804
20805
20806
20807
20808
20809
20810
20811
20812
20813
20814
20815
20816
20817
20818
20819
20820
20821
20822
20823
20824
20825
20826
20827
20828
20829
20830
20831
20832
20833
20834
20835
20836
20837
20838
20839
20840
20841
20842
20843
20844
20845
20846
20847
20848
20849
20850
20851
20852
20853
20854
20855
20856
20857
20858
20859
20860
20861
20862
20863
20864
20865
20866
20867
20868
20869
20870
20871
20872
20873
20874
20875
20876
20877
20878
20879
20880
20881
20882
20883
20884
20885
20886
20887
20888
20889
20890
20891
20892
20893
20894
20895
20896
20897
20898
20899
20900
20901
20902
20903
20904
20905
20906
20907
20908
20909
20910
20911
20912
20913
20914
20915
20916
20917
20918
20919
20920
20921
20922
20923
20924
20925
20926
20927
20928
20929
20930
20931
20932
20933
20934
20935
20936
20937
20938
20939
20940
20941
20942
20943
20944
20945
20946
20947
20948
20949
20950
20951
20952
20953
20954
20955
20956
20957
20958
20959
20960
20961
20962
20963
20964
20965
20966
20967
20968
20969
20970
20971
20972
20973
20974
20975
20976
20977
20978
20979
20980
20981
20982
20983
20984
20985
20986
20987
20988
20989
20990
20991
20992
20993
20994
20995
20996
20997
20998
20999
21000
21001
21002
21003
21004
21005
21006
21007
21008
21009
21010
21011
21012
21013
21014
21015
21016
21017
21018
21019
21020
21021
21022
21023
21024
21025
21026
21027
21028
21029
21030
21031
21032
21033
21034
21035
21036
21037
21038
21039
21040
21041
21042
21043
21044
21045
21046
21047
21048
21049
21050
21051
21052
21053
21054
21055
21056
21057
21058
21059
21060
21061
21062
21063
21064
21065
21066
21067
21068
21069
21070
21071
21072
21073
21074
21075
21076
21077
21078
21079
21080
21081
21082
21083
21084
21085
21086
21087
21088
21089
21090
21091
21092
21093
21094
21095
21096
21097
21098
21099
21100
21101
21102
21103
21104
21105
21106
21107
21108
21109
21110
21111
21112
21113
21114
21115
21116
21117
21118
21119
21120
21121
21122
21123
21124
21125
21126
21127
21128
21129
21130
21131
21132
21133
21134
21135
21136
21137
21138
21139
21140
21141
21142
21143
21144
21145
21146
21147
21148
21149
21150
21151
21152
21153
21154
21155
21156
21157
21158
21159
21160
21161
21162
21163
21164
21165
21166
21167
21168
21169
21170
21171
21172
21173
21174
21175
21176
21177
21178
21179
21180
21181
21182
21183
21184
21185
21186
21187
21188
21189
21190
21191
21192
21193
21194
21195
21196
21197
21198
21199
21200
21201
21202
21203
21204
21205
21206
21207
21208
21209
21210
21211
21212
21213
21214
21215
21216
21217
21218
21219
21220
21221
21222
21223
21224
21225
21226
21227
21228
21229
21230
21231
21232
21233
21234
21235
21236
21237
21238
21239
21240
21241
21242
21243
21244
21245
21246
21247
21248
21249
21250
21251
21252
21253
21254
21255
21256
21257
21258
21259
21260
21261
21262
21263
21264
21265
21266
21267
21268
21269
21270
21271
21272
21273
21274
21275
21276
21277
21278
21279
21280
21281
21282
21283
21284
21285
21286
21287
21288
21289
21290
21291
21292
21293
21294
21295
21296
21297
21298
21299
21300
21301
21302
21303
21304
21305
21306
21307
21308
21309
21310
21311
21312
21313
21314
21315
21316
21317
21318
21319
21320
21321
21322
21323
21324
21325
21326
21327
21328
21329
21330
21331
21332
21333
21334
21335
21336
21337
21338
21339
21340
21341
21342
21343
21344
21345
21346
21347
21348
21349
21350
21351
21352
21353
21354
21355
21356
21357
21358
21359
21360
21361
21362
21363
21364
21365
21366
21367
21368
21369
21370
21371
21372
21373
21374
21375
21376
21377
21378
21379
21380
21381
21382
21383
21384
21385
21386
21387
21388
21389
21390
21391
21392
21393
21394
21395
21396
21397
21398
21399
21400
21401
21402
21403
21404
21405
21406
21407
21408
21409
21410
21411
21412
21413
21414
21415
21416
21417
21418
21419
21420
21421
21422
21423
21424
21425
21426
21427
21428
21429
21430
21431
21432
21433
21434
21435
21436
21437
21438
21439
21440
21441
21442
21443
21444
21445
21446
21447
21448
21449
21450
21451
21452
21453
21454
21455
21456
21457
21458
21459
21460
21461
21462
21463
21464
21465
21466
21467
21468
21469
21470
21471
21472
21473
21474
21475
21476
21477
21478
21479
21480
21481
21482
21483
21484
21485
21486
21487
21488
21489
21490
21491
21492
21493
21494
21495
21496
21497
21498
21499
21500
21501
21502
21503
21504
21505
21506
21507
21508
21509
21510
21511
21512
21513
21514
21515
21516
21517
21518
21519
21520
21521
21522
21523
21524
21525
21526
21527
21528
21529
21530
21531
21532
21533
21534
21535
21536
21537
21538
21539
21540
21541
21542
21543
21544
21545
21546
21547
21548
21549
21550
21551
21552
21553
21554
21555
21556
21557
21558
21559
21560
21561
21562
21563
21564
21565
21566
21567
21568
21569
21570
21571
21572
21573
21574
21575
21576
21577
21578
21579
21580
21581
21582
21583
21584
21585
21586
21587
21588
21589
21590
21591
21592
21593
21594
21595
21596
21597
21598
21599
21600
21601
21602
21603
21604
21605
21606
21607
21608
21609
21610
21611
21612
21613
21614
21615
21616
21617
21618
21619
21620
21621
21622
21623
21624
21625
21626
21627
21628
21629
21630
21631
21632
21633
21634
21635
21636
21637
21638
21639
21640
21641
21642
21643
21644
21645
21646
21647
21648
21649
21650
21651
21652
21653
21654
21655
21656
21657
21658
21659
21660
21661
21662
21663
21664
21665
21666
21667
21668
21669
21670
21671
21672
21673
21674
21675
21676
21677
21678
21679
21680
21681
21682
21683
21684
21685
21686
21687
21688
21689
21690
21691
21692
21693
21694
21695
21696
21697
21698
21699
21700
21701
21702
21703
21704
21705
21706
21707
21708
21709
21710
21711
21712
21713
21714
21715
21716
21717
21718
21719
21720
21721
21722
21723
21724
21725
21726
21727
21728
21729
21730
21731
21732
21733
21734
21735
21736
21737
21738
21739
21740
21741
21742
21743
21744
21745
21746
21747
21748
21749
21750
21751
21752
21753
21754
21755
21756
21757
21758
21759
21760
21761
21762
21763
21764
21765
21766
21767
21768
21769
21770
21771
21772
21773
21774
21775
21776
21777
21778
21779
21780
21781
21782
21783
21784
21785
21786
21787
21788
21789
21790
21791
21792
21793
21794
21795
21796
21797
21798
21799
21800
21801
21802
21803
21804
21805
21806
21807
21808
21809
21810
21811
21812
21813
21814
21815
21816
21817
21818
21819
21820
21821
21822
21823
21824
21825
21826
21827
21828
21829
21830
21831
21832
21833
21834
21835
21836
21837
21838
21839
21840
21841
21842
21843
21844
21845
21846
21847
21848
21849
21850
21851
21852
21853
21854
21855
21856
21857
21858
21859
21860
21861
21862
21863
21864
21865
21866
21867
21868
21869
21870
21871
21872
21873
21874
21875
21876
21877
21878
21879
21880
21881
21882
21883
21884
21885
21886
21887
21888
21889
21890
21891
21892
21893
21894
21895
21896
21897
21898
21899
21900
21901
21902
21903
21904
21905
21906
21907
21908
21909
21910
21911
21912
21913
21914
21915
21916
21917
21918
21919
21920
21921
21922
21923
21924
21925
21926
21927
21928
21929
21930
21931
21932
21933
21934
21935
21936
21937
21938
21939
21940
21941
21942
21943
21944
21945
21946
21947
21948
21949
21950
21951
21952
21953
21954
21955
21956
21957
21958
21959
21960
21961
21962
21963
21964
21965
21966
21967
21968
21969
21970
21971
21972
21973
21974
21975
21976
21977
21978
21979
21980
21981
21982
21983
21984
21985
21986
21987
21988
21989
21990
21991
21992
21993
21994
21995
21996
21997
21998
21999
22000
22001
22002
22003
22004
22005
22006
22007
22008
22009
22010
22011
22012
22013
22014
22015
22016
22017
22018
22019
22020
22021
22022
22023
22024
22025
22026
22027
22028
22029
22030
22031
22032
22033
22034
22035
22036
22037
22038
22039
22040
22041
22042
22043
22044
22045
22046
22047
22048
22049
22050
22051
22052
22053
22054
22055
22056
22057
22058
22059
22060
22061
22062
22063
22064
22065
22066
22067
22068
22069
22070
22071
22072
22073
22074
22075
22076
22077
22078
22079
22080
22081
22082
22083
22084
22085
22086
22087
22088
22089
22090
22091
22092
22093
22094
22095
22096
22097
22098
22099
22100
22101
22102
22103
22104
22105
22106
22107
22108
22109
22110
22111
22112
22113
22114
22115
22116
22117
22118
22119
22120
22121
22122
22123
22124
22125
22126
22127
22128
22129
22130
22131
22132
22133
22134
22135
22136
22137
22138
22139
22140
22141
22142
22143
22144
22145
22146
22147
22148
22149
22150
22151
22152
22153
22154
22155
22156
22157
22158
22159
22160
22161
22162
22163
22164
22165
22166
22167
22168
22169
22170
22171
22172
22173
22174
22175
22176
22177
22178
22179
22180
22181
22182
22183
22184
22185
22186
22187
22188
22189
22190
22191
22192
22193
22194
22195
22196
22197
22198
22199
22200
22201
22202
22203
22204
22205
22206
22207
22208
22209
22210
22211
22212
22213
22214
22215
22216
22217
22218
22219
22220
22221
22222
22223
22224
22225
22226
22227
22228
22229
22230
22231
22232
22233
22234
22235
22236
22237
22238
22239
22240
22241
22242
22243
22244
22245
22246
22247
22248
22249
22250
22251
22252
22253
22254
22255
22256
22257
22258
22259
22260
22261
22262
22263
22264
22265
22266
22267
22268
22269
22270
22271
22272
22273
22274
22275
22276
22277
22278
22279
22280
22281
22282
22283
22284
22285
22286
22287
22288
22289
22290
22291
22292
22293
22294
22295
22296
22297
22298
22299
22300
22301
22302
22303
22304
22305
22306
22307
22308
22309
22310
22311
22312
22313
22314
22315
22316
22317
22318
22319
22320
22321
22322
22323
22324
22325
22326
22327
22328
22329
22330
22331
22332
22333
22334
22335
22336
22337
22338
22339
22340
22341
22342
22343
22344
22345
22346
22347
22348
22349
22350
22351
22352
22353
22354
22355
22356
22357
22358
22359
22360
22361
22362
22363
22364
22365
22366
22367
22368
22369
22370
22371
22372
22373
22374
22375
22376
22377
22378
22379
22380
22381
22382
22383
22384
22385
22386
22387
22388
22389
22390
22391
22392
22393
22394
22395
22396
22397
22398
22399
22400
22401
22402
22403
22404
22405
22406
22407
22408
22409
22410
22411
22412
22413
22414
22415
22416
22417
22418
22419
22420
22421
22422
22423
22424
22425
22426
22427
22428
22429
22430
22431
22432
22433
22434
22435
22436
22437
22438
22439
22440
22441
22442
22443
22444
22445
22446
22447
22448
22449
22450
22451
22452
22453
22454
22455
22456
22457
22458
22459
22460
22461
22462
22463
22464
22465
22466
22467
22468
22469
22470
22471
22472
22473
22474
22475
22476
22477
22478
22479
22480
22481
22482
22483
22484
22485
22486
22487
22488
22489
22490
22491
22492
22493
22494
22495
22496
22497
22498
22499
22500
22501
22502
22503
22504
22505
22506
22507
22508
22509
22510
22511
22512
22513
22514
22515
22516
22517
22518
22519
22520
22521
22522
22523
22524
22525
22526
22527
22528
22529
22530
22531
22532
22533
22534
22535
22536
22537
22538
22539
22540
22541
22542
22543
22544
22545
22546
22547
22548
22549
22550
22551
22552
22553
22554
22555
22556
22557
22558
22559
22560
22561
22562
22563
22564
22565
22566
22567
22568
22569
22570
22571
22572
22573
22574
22575
22576
22577
22578
22579
22580
22581
22582
22583
22584
22585
22586
22587
22588
22589
22590
22591
22592
22593
22594
22595
22596
22597
22598
22599
22600
22601
22602
22603
22604
22605
22606
22607
22608
22609
22610
22611
22612
22613
22614
22615
22616
22617
22618
22619
22620
22621
22622
22623
22624
22625
22626
22627
22628
22629
22630
22631
22632
22633
22634
22635
22636
22637
22638
22639
22640
22641
22642
22643
22644
22645
22646
22647
22648
22649
22650
22651
22652
22653
22654
22655
22656
22657
22658
22659
22660
22661
22662
22663
22664
22665
22666
22667
22668
22669
22670
22671
22672
22673
22674
22675
22676
22677
22678
22679
22680
22681
22682
22683
22684
22685
22686
22687
22688
22689
22690
22691
22692
22693
22694
22695
22696
22697
22698
22699
22700
22701
22702
22703
22704
22705
22706
22707
22708
22709
22710
22711
22712
22713
22714
22715
22716
22717
22718
22719
22720
22721
22722
22723
22724
22725
22726
22727
22728
22729
22730
22731
22732
22733
22734
22735
22736
22737
22738
22739
22740
22741
22742
22743
22744
22745
22746
22747
22748
22749
22750
22751
22752
22753
22754
22755
22756
22757
22758
22759
22760
22761
22762
22763
22764
22765
22766
22767
22768
22769
22770
22771
22772
22773
22774
22775
22776
22777
22778
22779
22780
22781
22782
22783
22784
22785
22786
22787
22788
22789
22790
22791
22792
22793
22794
22795
22796
22797
22798
22799
22800
22801
22802
22803
22804
22805
22806
22807
22808
22809
22810
22811
22812
22813
22814
22815
22816
22817
22818
22819
22820
22821
22822
22823
22824
22825
22826
22827
22828
22829
22830
22831
22832
22833
22834
22835
22836
22837
22838
22839
22840
22841
22842
22843
22844
22845
22846
22847
22848
22849
22850
22851
22852
22853
22854
22855
22856
22857
22858
22859
22860
22861
22862
22863
22864
22865
22866
22867
22868
22869
22870
22871
22872
22873
22874
22875
22876
22877
22878
22879
22880
22881
22882
22883
22884
22885
22886
22887
22888
22889
22890
22891
22892
22893
22894
22895
22896
22897
22898
22899
22900
22901
22902
22903
22904
22905
22906
22907
22908
22909
22910
22911
22912
22913
22914
22915
22916
22917
22918
22919
22920
22921
22922
22923
22924
22925
22926
22927
22928
22929
22930
22931
22932
22933
22934
22935
22936
22937
22938
22939
22940
22941
22942
22943
22944
22945
22946
22947
22948
22949
22950
22951
22952
22953
22954
22955
22956
22957
22958
22959
22960
22961
22962
22963
22964
22965
22966
22967
22968
22969
22970
22971
22972
22973
22974
22975
22976
22977
22978
22979
22980
22981
22982
22983
22984
22985
22986
22987
22988
22989
22990
22991
22992
22993
22994
22995
22996
22997
22998
22999
23000
23001
23002
23003
23004
23005
23006
23007
23008
23009
23010
23011
23012
23013
23014
23015
23016
23017
23018
23019
23020
23021
23022
23023
23024
23025
23026
23027
23028
23029
23030
23031
23032
23033
23034
23035
23036
23037
23038
23039
23040
23041
23042
23043
23044
23045
23046
23047
23048
23049
23050
23051
23052
23053
23054
23055
23056
23057
23058
23059
23060
23061
23062
23063
23064
23065
23066
23067
23068
23069
23070
23071
23072
23073
23074
23075
23076
23077
23078
23079
23080
23081
23082
23083
23084
23085
23086
23087
23088
23089
23090
23091
23092
23093
23094
23095
23096
23097
23098
23099
23100
23101
23102
23103
23104
23105
23106
23107
23108
23109
23110
23111
23112
23113
23114
23115
23116
23117
23118
23119
23120
23121
23122
23123
23124
23125
23126
23127
23128
23129
23130
23131
23132
23133
23134
23135
23136
23137
23138
23139
23140
23141
23142
23143
23144
23145
23146
23147
23148
23149
23150
23151
23152
23153
23154
23155
23156
23157
23158
23159
23160
23161
23162
23163
23164
23165
23166
23167
23168
23169
23170
23171
23172
23173
23174
23175
23176
23177
23178
23179
23180
23181
23182
23183
23184
23185
23186
23187
23188
23189
23190
23191
23192
23193
23194
23195
23196
23197
23198
23199
23200
23201
23202
23203
23204
23205
23206
23207
23208
23209
23210
23211
23212
23213
23214
23215
23216
23217
23218
23219
23220
23221
23222
23223
23224
23225
23226
23227
23228
23229
23230
23231
23232
23233
23234
23235
23236
23237
23238
23239
23240
23241
23242
23243
23244
23245
23246
23247
23248
23249
23250
23251
23252
23253
23254
23255
23256
23257
23258
23259
23260
23261
23262
23263
23264
23265
23266
23267
23268
23269
23270
23271
23272
23273
23274
23275
23276
23277
23278
23279
23280
23281
23282
23283
23284
23285
23286
23287
23288
23289
23290
23291
23292
23293
23294
23295
23296
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 * Copyright (c) 2016 Facebook
 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
 */
#include <uapi/linux/btf.h>
#include <linux/bpf-cgroup.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
#include <net/netlink.h>
#include <linux/file.h>
#include <linux/vmalloc.h>
#include <linux/stringify.h>
#include <linux/bsearch.h>
#include <linux/sort.h>
#include <linux/perf_event.h>
#include <linux/ctype.h>
#include <linux/error-injection.h>
#include <linux/bpf_lsm.h>
#include <linux/btf_ids.h>
#include <linux/poison.h>
#include <linux/module.h>
#include <linux/cpumask.h>
#include <linux/bpf_mem_alloc.h>
#include <net/xdp.h>
#include <linux/trace_events.h>
#include <linux/kallsyms.h>

#include "disasm.h"

static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
        [_id] = & _name ## _verifier_ops,
#define BPF_MAP_TYPE(_id, _ops)
#define BPF_LINK_TYPE(_id, _name)
#include <linux/bpf_types.h>
#undef BPF_PROG_TYPE
#undef BPF_MAP_TYPE
#undef BPF_LINK_TYPE
};

struct bpf_mem_alloc bpf_global_percpu_ma;
static bool bpf_global_percpu_ma_set;

/* bpf_check() is a static code analyzer that walks eBPF program
 * instruction by instruction and updates register/stack state.
 * All paths of conditional branches are analyzed until 'bpf_exit' insn.
 *
 * The first pass is depth-first-search to check that the program is a DAG.
 * It rejects the following programs:
 * - larger than BPF_MAXINSNS insns
 * - if loop is present (detected via back-edge)
 * - unreachable insns exist (shouldn't be a forest. program = one function)
 * - out of bounds or malformed jumps
 * The second pass is all possible path descent from the 1st insn.
 * Since it's analyzing all paths through the program, the length of the
 * analysis is limited to 64k insn, which may be hit even if total number of
 * insn is less then 4K, but there are too many branches that change stack/regs.
 * Number of 'branches to be analyzed' is limited to 1k
 *
 * On entry to each instruction, each register has a type, and the instruction
 * changes the types of the registers depending on instruction semantics.
 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
 * copied to R1.
 *
 * All registers are 64-bit.
 * R0 - return register
 * R1-R5 argument passing registers
 * R6-R9 callee saved registers
 * R10 - frame pointer read-only
 *
 * At the start of BPF program the register R1 contains a pointer to bpf_context
 * and has type PTR_TO_CTX.
 *
 * Verifier tracks arithmetic operations on pointers in case:
 *    BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
 *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
 * 1st insn copies R10 (which has FRAME_PTR) type into R1
 * and 2nd arithmetic instruction is pattern matched to recognize
 * that it wants to construct a pointer to some element within stack.
 * So after 2nd insn, the register R1 has type PTR_TO_STACK
 * (and -20 constant is saved for further stack bounds checking).
 * Meaning that this reg is a pointer to stack plus known immediate constant.
 *
 * Most of the time the registers have SCALAR_VALUE type, which
 * means the register has some value, but it's not a valid pointer.
 * (like pointer plus pointer becomes SCALAR_VALUE type)
 *
 * When verifier sees load or store instructions the type of base register
 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
 * four pointer types recognized by check_mem_access() function.
 *
 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
 * and the range of [ptr, ptr + map's value_size) is accessible.
 *
 * registers used to pass values to function calls are checked against
 * function argument constraints.
 *
 * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
 * It means that the register type passed to this function must be
 * PTR_TO_STACK and it will be used inside the function as
 * 'pointer to map element key'
 *
 * For example the argument constraints for bpf_map_lookup_elem():
 *   .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
 *   .arg1_type = ARG_CONST_MAP_PTR,
 *   .arg2_type = ARG_PTR_TO_MAP_KEY,
 *
 * ret_type says that this function returns 'pointer to map elem value or null'
 * function expects 1st argument to be a const pointer to 'struct bpf_map' and
 * 2nd argument should be a pointer to stack, which will be used inside
 * the helper function as a pointer to map element key.
 *
 * On the kernel side the helper function looks like:
 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
 * {
 *    struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
 *    void *key = (void *) (unsigned long) r2;
 *    void *value;
 *
 *    here kernel can access 'key' and 'map' pointers safely, knowing that
 *    [key, key + map->key_size) bytes are valid and were initialized on
 *    the stack of eBPF program.
 * }
 *
 * Corresponding eBPF program may look like:
 *    BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),  // after this insn R2 type is FRAME_PTR
 *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
 *    BPF_LD_MAP_FD(BPF_REG_1, map_fd),      // after this insn R1 type is CONST_PTR_TO_MAP
 *    BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
 * here verifier looks at prototype of map_lookup_elem() and sees:
 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
 *
 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
 * and were initialized prior to this call.
 * If it's ok, then verifier allows this BPF_CALL insn and looks at
 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
 * returns either pointer to map value or NULL.
 *
 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
 * insn, the register holding that pointer in the true branch changes state to
 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
 * branch. See check_cond_jmp_op().
 *
 * After the call R0 is set to return type of the function and registers R1-R5
 * are set to NOT_INIT to indicate that they are no longer readable.
 *
 * The following reference types represent a potential reference to a kernel
 * resource which, after first being allocated, must be checked and freed by
 * the BPF program:
 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
 *
 * When the verifier sees a helper call return a reference type, it allocates a
 * pointer id for the reference and stores it in the current function state.
 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
 * passes through a NULL-check conditional. For the branch wherein the state is
 * changed to CONST_IMM, the verifier releases the reference.
 *
 * For each helper function that allocates a reference, such as
 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
 * bpf_sk_release(). When a reference type passes into the release function,
 * the verifier also releases the reference. If any unchecked or unreleased
 * reference remains at the end of the program, the verifier rejects it.
 */

/* verifier_state + insn_idx are pushed to stack when branch is encountered */
struct bpf_verifier_stack_elem {
        /* verifier state is 'st'
         * before processing instruction 'insn_idx'
         * and after processing instruction 'prev_insn_idx'
         */
        struct bpf_verifier_state st;
        int insn_idx;
        int prev_insn_idx;
        struct bpf_verifier_stack_elem *next;
        /* length of verifier log at the time this state was pushed on stack */
        u32 log_pos;
};

#define BPF_COMPLEXITY_LIMIT_JMP_SEQ        8192
#define BPF_COMPLEXITY_LIMIT_STATES        64

#define BPF_MAP_KEY_POISON        (1ULL << 63)
#define BPF_MAP_KEY_SEEN        (1ULL << 62)

#define BPF_GLOBAL_PERCPU_MA_MAX_SIZE  512

#define BPF_PRIV_STACK_MIN_SIZE                64

static int acquire_reference(struct bpf_verifier_env *env, int insn_idx);
static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id);
static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
static int ref_set_non_owning(struct bpf_verifier_env *env,
                              struct bpf_reg_state *reg);
static void specialize_kfunc(struct bpf_verifier_env *env,
                             u32 func_id, u16 offset, unsigned long *addr);
static bool is_trusted_reg(const struct bpf_reg_state *reg);

static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
{
        return aux->map_ptr_state.poison;
}

static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
{
        return aux->map_ptr_state.unpriv;
}

static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
                              struct bpf_map *map,
                              bool unpriv, bool poison)
{
        unpriv |= bpf_map_ptr_unpriv(aux);
        aux->map_ptr_state.unpriv = unpriv;
        aux->map_ptr_state.poison = poison;
        aux->map_ptr_state.map_ptr = map;
}

static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
{
        return aux->map_key_state & BPF_MAP_KEY_POISON;
}

static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
{
        return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
}

static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
{
        return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
}

static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state)
{
        bool poisoned = bpf_map_key_poisoned(aux);

        aux->map_key_state = state | BPF_MAP_KEY_SEEN |
                             (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
}

static bool bpf_helper_call(const struct bpf_insn *insn)
{
        return insn->code == (BPF_JMP | BPF_CALL) &&
               insn->src_reg == 0;
}

static bool bpf_pseudo_call(const struct bpf_insn *insn)
{
        return insn->code == (BPF_JMP | BPF_CALL) &&
               insn->src_reg == BPF_PSEUDO_CALL;
}

static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
{
        return insn->code == (BPF_JMP | BPF_CALL) &&
               insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
}

struct bpf_call_arg_meta {
        struct bpf_map *map_ptr;
        bool raw_mode;
        bool pkt_access;
        u8 release_regno;
        int regno;
        int access_size;
        int mem_size;
        u64 msize_max_value;
        int ref_obj_id;
        int dynptr_id;
        int map_uid;
        int func_id;
        struct btf *btf;
        u32 btf_id;
        struct btf *ret_btf;
        u32 ret_btf_id;
        u32 subprogno;
        struct btf_field *kptr_field;
        s64 const_map_key;
};

struct bpf_kfunc_call_arg_meta {
        /* In parameters */
        struct btf *btf;
        u32 func_id;
        u32 kfunc_flags;
        const struct btf_type *func_proto;
        const char *func_name;
        /* Out parameters */
        u32 ref_obj_id;
        u8 release_regno;
        bool r0_rdonly;
        u32 ret_btf_id;
        u64 r0_size;
        u32 subprogno;
        struct {
                u64 value;
                bool found;
        } arg_constant;

        /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling,
         * generally to pass info about user-defined local kptr types to later
         * verification logic
         *   bpf_obj_drop/bpf_percpu_obj_drop
         *     Record the local kptr type to be drop'd
         *   bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type)
         *     Record the local kptr type to be refcount_incr'd and use
         *     arg_owning_ref to determine whether refcount_acquire should be
         *     fallible
         */
        struct btf *arg_btf;
        u32 arg_btf_id;
        bool arg_owning_ref;

        struct {
                struct btf_field *field;
        } arg_list_head;
        struct {
                struct btf_field *field;
        } arg_rbtree_root;
        struct {
                enum bpf_dynptr_type type;
                u32 id;
                u32 ref_obj_id;
        } initialized_dynptr;
        struct {
                u8 spi;
                u8 frameno;
        } iter;
        struct {
                struct bpf_map *ptr;
                int uid;
        } map;
        u64 mem_size;
};

struct btf *btf_vmlinux;

static const char *btf_type_name(const struct btf *btf, u32 id)
{
        return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
}

static DEFINE_MUTEX(bpf_verifier_lock);
static DEFINE_MUTEX(bpf_percpu_ma_lock);

__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
{
        struct bpf_verifier_env *env = private_data;
        va_list args;

        if (!bpf_verifier_log_needed(&env->log))
                return;

        va_start(args, fmt);
        bpf_verifier_vlog(&env->log, fmt, args);
        va_end(args);
}

static void verbose_invalid_scalar(struct bpf_verifier_env *env,
                                   struct bpf_reg_state *reg,
                                   struct bpf_retval_range range, const char *ctx,
                                   const char *reg_name)
{
        bool unknown = true;

        verbose(env, "%s the register %s has", ctx, reg_name);
        if (reg->smin_value > S64_MIN) {
                verbose(env, " smin=%lld", reg->smin_value);
                unknown = false;
        }
        if (reg->smax_value < S64_MAX) {
                verbose(env, " smax=%lld", reg->smax_value);
                unknown = false;
        }
        if (unknown)
                verbose(env, " unknown scalar value");
        verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);
}

static bool reg_not_null(const struct bpf_reg_state *reg)
{
        enum bpf_reg_type type;

        type = reg->type;
        if (type_may_be_null(type))
                return false;

        type = base_type(type);
        return type == PTR_TO_SOCKET ||
                type == PTR_TO_TCP_SOCK ||
                type == PTR_TO_MAP_VALUE ||
                type == PTR_TO_MAP_KEY ||
                type == PTR_TO_SOCK_COMMON ||
                (type == PTR_TO_BTF_ID && is_trusted_reg(reg)) ||
                type == PTR_TO_MEM;
}

static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg)
{
        struct btf_record *rec = NULL;
        struct btf_struct_meta *meta;

        if (reg->type == PTR_TO_MAP_VALUE) {
                rec = reg->map_ptr->record;
        } else if (type_is_ptr_alloc_obj(reg->type)) {
                meta = btf_find_struct_meta(reg->btf, reg->btf_id);
                if (meta)
                        rec = meta->record;
        }
        return rec;
}

static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)
{
        struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;

        return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
}

static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)
{
        struct bpf_func_info *info;

        if (!env->prog->aux->func_info)
                return "";

        info = &env->prog->aux->func_info[subprog];
        return btf_type_name(env->prog->aux->btf, info->type_id);
}

static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)
{
        struct bpf_subprog_info *info = subprog_info(env, subprog);

        info->is_cb = true;
        info->is_async_cb = true;
        info->is_exception_cb = true;
}

static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog)
{
        return subprog_info(env, subprog)->is_exception_cb;
}

static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
{
        return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK);
}

static bool type_is_rdonly_mem(u32 type)
{
        return type & MEM_RDONLY;
}

static bool is_acquire_function(enum bpf_func_id func_id,
                                const struct bpf_map *map)
{
        enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC;

        if (func_id == BPF_FUNC_sk_lookup_tcp ||
            func_id == BPF_FUNC_sk_lookup_udp ||
            func_id == BPF_FUNC_skc_lookup_tcp ||
            func_id == BPF_FUNC_ringbuf_reserve ||
            func_id == BPF_FUNC_kptr_xchg)
                return true;

        if (func_id == BPF_FUNC_map_lookup_elem &&
            (map_type == BPF_MAP_TYPE_SOCKMAP ||
             map_type == BPF_MAP_TYPE_SOCKHASH))
                return true;

        return false;
}

static bool is_ptr_cast_function(enum bpf_func_id func_id)
{
        return func_id == BPF_FUNC_tcp_sock ||
                func_id == BPF_FUNC_sk_fullsock ||
                func_id == BPF_FUNC_skc_to_tcp_sock ||
                func_id == BPF_FUNC_skc_to_tcp6_sock ||
                func_id == BPF_FUNC_skc_to_udp6_sock ||
                func_id == BPF_FUNC_skc_to_mptcp_sock ||
                func_id == BPF_FUNC_skc_to_tcp_timewait_sock ||
                func_id == BPF_FUNC_skc_to_tcp_request_sock;
}

static bool is_dynptr_ref_function(enum bpf_func_id func_id)
{
        return func_id == BPF_FUNC_dynptr_data;
}

static bool is_sync_callback_calling_kfunc(u32 btf_id);
static bool is_async_callback_calling_kfunc(u32 btf_id);
static bool is_callback_calling_kfunc(u32 btf_id);
static bool is_bpf_throw_kfunc(struct bpf_insn *insn);

static bool is_bpf_wq_set_callback_impl_kfunc(u32 btf_id);

static bool is_sync_callback_calling_function(enum bpf_func_id func_id)
{
        return func_id == BPF_FUNC_for_each_map_elem ||
               func_id == BPF_FUNC_find_vma ||
               func_id == BPF_FUNC_loop ||
               func_id == BPF_FUNC_user_ringbuf_drain;
}

static bool is_async_callback_calling_function(enum bpf_func_id func_id)
{
        return func_id == BPF_FUNC_timer_set_callback;
}

static bool is_callback_calling_function(enum bpf_func_id func_id)
{
        return is_sync_callback_calling_function(func_id) ||
               is_async_callback_calling_function(func_id);
}

static bool is_sync_callback_calling_insn(struct bpf_insn *insn)
{
        return (bpf_helper_call(insn) && is_sync_callback_calling_function(insn->imm)) ||
               (bpf_pseudo_kfunc_call(insn) && is_sync_callback_calling_kfunc(insn->imm));
}

static bool is_async_callback_calling_insn(struct bpf_insn *insn)
{
        return (bpf_helper_call(insn) && is_async_callback_calling_function(insn->imm)) ||
               (bpf_pseudo_kfunc_call(insn) && is_async_callback_calling_kfunc(insn->imm));
}

static bool is_may_goto_insn(struct bpf_insn *insn)
{
        return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO;
}

static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)
{
        return is_may_goto_insn(&env->prog->insnsi[insn_idx]);
}

static bool is_storage_get_function(enum bpf_func_id func_id)
{
        return func_id == BPF_FUNC_sk_storage_get ||
               func_id == BPF_FUNC_inode_storage_get ||
               func_id == BPF_FUNC_task_storage_get ||
               func_id == BPF_FUNC_cgrp_storage_get;
}

static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id,
                                        const struct bpf_map *map)
{
        int ref_obj_uses = 0;

        if (is_ptr_cast_function(func_id))
                ref_obj_uses++;
        if (is_acquire_function(func_id, map))
                ref_obj_uses++;
        if (is_dynptr_ref_function(func_id))
                ref_obj_uses++;

        return ref_obj_uses > 1;
}

static bool is_cmpxchg_insn(const struct bpf_insn *insn)
{
        return BPF_CLASS(insn->code) == BPF_STX &&
               BPF_MODE(insn->code) == BPF_ATOMIC &&
               insn->imm == BPF_CMPXCHG;
}

static int __get_spi(s32 off)
{
        return (-off - 1) / BPF_REG_SIZE;
}

static struct bpf_func_state *func(struct bpf_verifier_env *env,
                                   const struct bpf_reg_state *reg)
{
        struct bpf_verifier_state *cur = env->cur_state;

        return cur->frame[reg->frameno];
}

static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)
{
       int allocated_slots = state->allocated_stack / BPF_REG_SIZE;

       /* We need to check that slots between [spi - nr_slots + 1, spi] are
        * within [0, allocated_stack).
        *
        * Please note that the spi grows downwards. For example, a dynptr
        * takes the size of two stack slots; the first slot will be at
        * spi and the second slot will be at spi - 1.
        */
       return spi - nr_slots + 1 >= 0 && spi < allocated_slots;
}

static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
                                  const char *obj_kind, int nr_slots)
{
        int off, spi;

        if (!tnum_is_const(reg->var_off)) {
                verbose(env, "%s has to be at a constant offset\n", obj_kind);
                return -EINVAL;
        }

        off = reg->off + reg->var_off.value;
        if (off % BPF_REG_SIZE) {
                verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
                return -EINVAL;
        }

        spi = __get_spi(off);
        if (spi + 1 < nr_slots) {
                verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
                return -EINVAL;
        }

        if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots))
                return -ERANGE;
        return spi;
}

static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS);
}

static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots)
{
        return stack_slot_obj_get_spi(env, reg, "iter", nr_slots);
}

static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        return stack_slot_obj_get_spi(env, reg, "irq_flag", 1);
}

static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
{
        switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
        case DYNPTR_TYPE_LOCAL:
                return BPF_DYNPTR_TYPE_LOCAL;
        case DYNPTR_TYPE_RINGBUF:
                return BPF_DYNPTR_TYPE_RINGBUF;
        case DYNPTR_TYPE_SKB:
                return BPF_DYNPTR_TYPE_SKB;
        case DYNPTR_TYPE_XDP:
                return BPF_DYNPTR_TYPE_XDP;
        default:
                return BPF_DYNPTR_TYPE_INVALID;
        }
}

static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type)
{
        switch (type) {
        case BPF_DYNPTR_TYPE_LOCAL:
                return DYNPTR_TYPE_LOCAL;
        case BPF_DYNPTR_TYPE_RINGBUF:
                return DYNPTR_TYPE_RINGBUF;
        case BPF_DYNPTR_TYPE_SKB:
                return DYNPTR_TYPE_SKB;
        case BPF_DYNPTR_TYPE_XDP:
                return DYNPTR_TYPE_XDP;
        default:
                return 0;
        }
}

static bool dynptr_type_refcounted(enum bpf_dynptr_type type)
{
        return type == BPF_DYNPTR_TYPE_RINGBUF;
}

static void __mark_dynptr_reg(struct bpf_reg_state *reg,
                              enum bpf_dynptr_type type,
                              bool first_slot, int dynptr_id);

static void __mark_reg_not_init(const struct bpf_verifier_env *env,
                                struct bpf_reg_state *reg);

static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,
                                   struct bpf_reg_state *sreg1,
                                   struct bpf_reg_state *sreg2,
                                   enum bpf_dynptr_type type)
{
        int id = ++env->id_gen;

        __mark_dynptr_reg(sreg1, type, true, id);
        __mark_dynptr_reg(sreg2, type, false, id);
}

static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,
                               struct bpf_reg_state *reg,
                               enum bpf_dynptr_type type)
{
        __mark_dynptr_reg(reg, type, true, ++env->id_gen);
}

static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
                                        struct bpf_func_state *state, int spi);

static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
                                   enum bpf_arg_type arg_type, int insn_idx, int clone_ref_obj_id)
{
        struct bpf_func_state *state = func(env, reg);
        enum bpf_dynptr_type type;
        int spi, i, err;

        spi = dynptr_get_spi(env, reg);
        if (spi < 0)
                return spi;

        /* We cannot assume both spi and spi - 1 belong to the same dynptr,
         * hence we need to call destroy_if_dynptr_stack_slot twice for both,
         * to ensure that for the following example:
         *        [d1][d1][d2][d2]
         * spi    3   2   1   0
         * So marking spi = 2 should lead to destruction of both d1 and d2. In
         * case they do belong to same dynptr, second call won't see slot_type
         * as STACK_DYNPTR and will simply skip destruction.
         */
        err = destroy_if_dynptr_stack_slot(env, state, spi);
        if (err)
                return err;
        err = destroy_if_dynptr_stack_slot(env, state, spi - 1);
        if (err)
                return err;

        for (i = 0; i < BPF_REG_SIZE; i++) {
                state->stack[spi].slot_type[i] = STACK_DYNPTR;
                state->stack[spi - 1].slot_type[i] = STACK_DYNPTR;
        }

        type = arg_to_dynptr_type(arg_type);
        if (type == BPF_DYNPTR_TYPE_INVALID)
                return -EINVAL;

        mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,
                               &state->stack[spi - 1].spilled_ptr, type);

        if (dynptr_type_refcounted(type)) {
                /* The id is used to track proper releasing */
                int id;

                if (clone_ref_obj_id)
                        id = clone_ref_obj_id;
                else
                        id = acquire_reference(env, insn_idx);

                if (id < 0)
                        return id;

                state->stack[spi].spilled_ptr.ref_obj_id = id;
                state->stack[spi - 1].spilled_ptr.ref_obj_id = id;
        }

        state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
        state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN;

        return 0;
}

static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi)
{
        int i;

        for (i = 0; i < BPF_REG_SIZE; i++) {
                state->stack[spi].slot_type[i] = STACK_INVALID;
                state->stack[spi - 1].slot_type[i] = STACK_INVALID;
        }

        __mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
        __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);

        /* Why do we need to set REG_LIVE_WRITTEN for STACK_INVALID slot?
         *
         * While we don't allow reading STACK_INVALID, it is still possible to
         * do <8 byte writes marking some but not all slots as STACK_MISC. Then,
         * helpers or insns can do partial read of that part without failing,
         * but check_stack_range_initialized, check_stack_read_var_off, and
         * check_stack_read_fixed_off will do mark_reg_read for all 8-bytes of
         * the slot conservatively. Hence we need to prevent those liveness
         * marking walks.
         *
         * This was not a problem before because STACK_INVALID is only set by
         * default (where the default reg state has its reg->parent as NULL), or
         * in clean_live_states after REG_LIVE_DONE (at which point
         * mark_reg_read won't walk reg->parent chain), but not randomly during
         * verifier state exploration (like we did above). Hence, for our case
         * parentage chain will still be live (i.e. reg->parent may be
         * non-NULL), while earlier reg->parent was NULL, so we need
         * REG_LIVE_WRITTEN to screen off read marker propagation when it is
         * done later on reads or by mark_dynptr_read as well to unnecessary
         * mark registers in verifier state.
         */
        state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
        state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN;
}

static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        struct bpf_func_state *state = func(env, reg);
        int spi, ref_obj_id, i;

        spi = dynptr_get_spi(env, reg);
        if (spi < 0)
                return spi;

        if (!dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
                invalidate_dynptr(env, state, spi);
                return 0;
        }

        ref_obj_id = state->stack[spi].spilled_ptr.ref_obj_id;

        /* If the dynptr has a ref_obj_id, then we need to invalidate
         * two things:
         *
         * 1) Any dynptrs with a matching ref_obj_id (clones)
         * 2) Any slices derived from this dynptr.
         */

        /* Invalidate any slices associated with this dynptr */
        WARN_ON_ONCE(release_reference(env, ref_obj_id));

        /* Invalidate any dynptr clones */
        for (i = 1; i < state->allocated_stack / BPF_REG_SIZE; i++) {
                if (state->stack[i].spilled_ptr.ref_obj_id != ref_obj_id)
                        continue;

                /* it should always be the case that if the ref obj id
                 * matches then the stack slot also belongs to a
                 * dynptr
                 */
                if (state->stack[i].slot_type[0] != STACK_DYNPTR) {
                        verbose(env, "verifier internal error: misconfigured ref_obj_id\n");
                        return -EFAULT;
                }
                if (state->stack[i].spilled_ptr.dynptr.first_slot)
                        invalidate_dynptr(env, state, i);
        }

        return 0;
}

static void __mark_reg_unknown(const struct bpf_verifier_env *env,
                               struct bpf_reg_state *reg);

static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        if (!env->allow_ptr_leaks)
                __mark_reg_not_init(env, reg);
        else
                __mark_reg_unknown(env, reg);
}

static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
                                        struct bpf_func_state *state, int spi)
{
        struct bpf_func_state *fstate;
        struct bpf_reg_state *dreg;
        int i, dynptr_id;

        /* We always ensure that STACK_DYNPTR is never set partially,
         * hence just checking for slot_type[0] is enough. This is
         * different for STACK_SPILL, where it may be only set for
         * 1 byte, so code has to use is_spilled_reg.
         */
        if (state->stack[spi].slot_type[0] != STACK_DYNPTR)
                return 0;

        /* Reposition spi to first slot */
        if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
                spi = spi + 1;

        if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
                verbose(env, "cannot overwrite referenced dynptr\n");
                return -EINVAL;
        }

        mark_stack_slot_scratched(env, spi);
        mark_stack_slot_scratched(env, spi - 1);

        /* Writing partially to one dynptr stack slot destroys both. */
        for (i = 0; i < BPF_REG_SIZE; i++) {
                state->stack[spi].slot_type[i] = STACK_INVALID;
                state->stack[spi - 1].slot_type[i] = STACK_INVALID;
        }

        dynptr_id = state->stack[spi].spilled_ptr.id;
        /* Invalidate any slices associated with this dynptr */
        bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({
                /* Dynptr slices are only PTR_TO_MEM_OR_NULL and PTR_TO_MEM */
                if (dreg->type != (PTR_TO_MEM | PTR_MAYBE_NULL) && dreg->type != PTR_TO_MEM)
                        continue;
                if (dreg->dynptr_id == dynptr_id)
                        mark_reg_invalid(env, dreg);
        }));

        /* Do not release reference state, we are destroying dynptr on stack,
         * not using some helper to release it. Just reset register.
         */
        __mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
        __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);

        /* Same reason as unmark_stack_slots_dynptr above */
        state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
        state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN;

        return 0;
}

static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        int spi;

        if (reg->type == CONST_PTR_TO_DYNPTR)
                return false;

        spi = dynptr_get_spi(env, reg);

        /* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an
         * error because this just means the stack state hasn't been updated yet.
         * We will do check_mem_access to check and update stack bounds later.
         */
        if (spi < 0 && spi != -ERANGE)
                return false;

        /* We don't need to check if the stack slots are marked by previous
         * dynptr initializations because we allow overwriting existing unreferenced
         * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls
         * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are
         * touching are completely destructed before we reinitialize them for a new
         * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early
         * instead of delaying it until the end where the user will get "Unreleased
         * reference" error.
         */
        return true;
}

static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        struct bpf_func_state *state = func(env, reg);
        int i, spi;

        /* This already represents first slot of initialized bpf_dynptr.
         *
         * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to
         * check_func_arg_reg_off's logic, so we don't need to check its
         * offset and alignment.
         */
        if (reg->type == CONST_PTR_TO_DYNPTR)
                return true;

        spi = dynptr_get_spi(env, reg);
        if (spi < 0)
                return false;
        if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
                return false;

        for (i = 0; i < BPF_REG_SIZE; i++) {
                if (state->stack[spi].slot_type[i] != STACK_DYNPTR ||
                    state->stack[spi - 1].slot_type[i] != STACK_DYNPTR)
                        return false;
        }

        return true;
}

static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
                                    enum bpf_arg_type arg_type)
{
        struct bpf_func_state *state = func(env, reg);
        enum bpf_dynptr_type dynptr_type;
        int spi;

        /* ARG_PTR_TO_DYNPTR takes any type of dynptr */
        if (arg_type == ARG_PTR_TO_DYNPTR)
                return true;

        dynptr_type = arg_to_dynptr_type(arg_type);
        if (reg->type == CONST_PTR_TO_DYNPTR) {
                return reg->dynptr.type == dynptr_type;
        } else {
                spi = dynptr_get_spi(env, reg);
                if (spi < 0)
                        return false;
                return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type;
        }
}

static void __mark_reg_known_zero(struct bpf_reg_state *reg);

static bool in_rcu_cs(struct bpf_verifier_env *env);

static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta);

static int mark_stack_slots_iter(struct bpf_verifier_env *env,
                                 struct bpf_kfunc_call_arg_meta *meta,
                                 struct bpf_reg_state *reg, int insn_idx,
                                 struct btf *btf, u32 btf_id, int nr_slots)
{
        struct bpf_func_state *state = func(env, reg);
        int spi, i, j, id;

        spi = iter_get_spi(env, reg, nr_slots);
        if (spi < 0)
                return spi;

        id = acquire_reference(env, insn_idx);
        if (id < 0)
                return id;

        for (i = 0; i < nr_slots; i++) {
                struct bpf_stack_state *slot = &state->stack[spi - i];
                struct bpf_reg_state *st = &slot->spilled_ptr;

                __mark_reg_known_zero(st);
                st->type = PTR_TO_STACK; /* we don't have dedicated reg type */
                if (is_kfunc_rcu_protected(meta)) {
                        if (in_rcu_cs(env))
                                st->type |= MEM_RCU;
                        else
                                st->type |= PTR_UNTRUSTED;
                }
                st->live |= REG_LIVE_WRITTEN;
                st->ref_obj_id = i == 0 ? id : 0;
                st->iter.btf = btf;
                st->iter.btf_id = btf_id;
                st->iter.state = BPF_ITER_STATE_ACTIVE;
                st->iter.depth = 0;

                for (j = 0; j < BPF_REG_SIZE; j++)
                        slot->slot_type[j] = STACK_ITER;

                mark_stack_slot_scratched(env, spi - i);
        }

        return 0;
}

static int unmark_stack_slots_iter(struct bpf_verifier_env *env,
                                   struct bpf_reg_state *reg, int nr_slots)
{
        struct bpf_func_state *state = func(env, reg);
        int spi, i, j;

        spi = iter_get_spi(env, reg, nr_slots);
        if (spi < 0)
                return spi;

        for (i = 0; i < nr_slots; i++) {
                struct bpf_stack_state *slot = &state->stack[spi - i];
                struct bpf_reg_state *st = &slot->spilled_ptr;

                if (i == 0)
                        WARN_ON_ONCE(release_reference(env, st->ref_obj_id));

                __mark_reg_not_init(env, st);

                /* see unmark_stack_slots_dynptr() for why we need to set REG_LIVE_WRITTEN */
                st->live |= REG_LIVE_WRITTEN;

                for (j = 0; j < BPF_REG_SIZE; j++)
                        slot->slot_type[j] = STACK_INVALID;

                mark_stack_slot_scratched(env, spi - i);
        }

        return 0;
}

static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env,
                                     struct bpf_reg_state *reg, int nr_slots)
{
        struct bpf_func_state *state = func(env, reg);
        int spi, i, j;

        /* For -ERANGE (i.e. spi not falling into allocated stack slots), we
         * will do check_mem_access to check and update stack bounds later, so
         * return true for that case.
         */
        spi = iter_get_spi(env, reg, nr_slots);
        if (spi == -ERANGE)
                return true;
        if (spi < 0)
                return false;

        for (i = 0; i < nr_slots; i++) {
                struct bpf_stack_state *slot = &state->stack[spi - i];

                for (j = 0; j < BPF_REG_SIZE; j++)
                        if (slot->slot_type[j] == STACK_ITER)
                                return false;
        }

        return true;
}

static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
                                   struct btf *btf, u32 btf_id, int nr_slots)
{
        struct bpf_func_state *state = func(env, reg);
        int spi, i, j;

        spi = iter_get_spi(env, reg, nr_slots);
        if (spi < 0)
                return -EINVAL;

        for (i = 0; i < nr_slots; i++) {
                struct bpf_stack_state *slot = &state->stack[spi - i];
                struct bpf_reg_state *st = &slot->spilled_ptr;

                if (st->type & PTR_UNTRUSTED)
                        return -EPROTO;
                /* only main (first) slot has ref_obj_id set */
                if (i == 0 && !st->ref_obj_id)
                        return -EINVAL;
                if (i != 0 && st->ref_obj_id)
                        return -EINVAL;
                if (st->iter.btf != btf || st->iter.btf_id != btf_id)
                        return -EINVAL;

                for (j = 0; j < BPF_REG_SIZE; j++)
                        if (slot->slot_type[j] != STACK_ITER)
                                return -EINVAL;
        }

        return 0;
}

static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);
static int release_irq_state(struct bpf_verifier_state *state, int id);

static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env,
                                     struct bpf_kfunc_call_arg_meta *meta,
                                     struct bpf_reg_state *reg, int insn_idx)
{
        struct bpf_func_state *state = func(env, reg);
        struct bpf_stack_state *slot;
        struct bpf_reg_state *st;
        int spi, i, id;

        spi = irq_flag_get_spi(env, reg);
        if (spi < 0)
                return spi;

        id = acquire_irq_state(env, insn_idx);
        if (id < 0)
                return id;

        slot = &state->stack[spi];
        st = &slot->spilled_ptr;

        __mark_reg_known_zero(st);
        st->type = PTR_TO_STACK; /* we don't have dedicated reg type */
        st->live |= REG_LIVE_WRITTEN;
        st->ref_obj_id = id;

        for (i = 0; i < BPF_REG_SIZE; i++)
                slot->slot_type[i] = STACK_IRQ_FLAG;

        mark_stack_slot_scratched(env, spi);
        return 0;
}

static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        struct bpf_func_state *state = func(env, reg);
        struct bpf_stack_state *slot;
        struct bpf_reg_state *st;
        int spi, i, err;

        spi = irq_flag_get_spi(env, reg);
        if (spi < 0)
                return spi;

        slot = &state->stack[spi];
        st = &slot->spilled_ptr;

        err = release_irq_state(env->cur_state, st->ref_obj_id);
        WARN_ON_ONCE(err && err != -EACCES);
        if (err) {
                int insn_idx = 0;

                for (int i = 0; i < env->cur_state->acquired_refs; i++) {
                        if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) {
                                insn_idx = env->cur_state->refs[i].insn_idx;
                                break;
                        }
                }

                verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n",
                        env->cur_state->active_irq_id, insn_idx);
                return err;
        }

        __mark_reg_not_init(env, st);

        /* see unmark_stack_slots_dynptr() for why we need to set REG_LIVE_WRITTEN */
        st->live |= REG_LIVE_WRITTEN;

        for (i = 0; i < BPF_REG_SIZE; i++)
                slot->slot_type[i] = STACK_INVALID;

        mark_stack_slot_scratched(env, spi);
        return 0;
}

static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        struct bpf_func_state *state = func(env, reg);
        struct bpf_stack_state *slot;
        int spi, i;

        /* For -ERANGE (i.e. spi not falling into allocated stack slots), we
         * will do check_mem_access to check and update stack bounds later, so
         * return true for that case.
         */
        spi = irq_flag_get_spi(env, reg);
        if (spi == -ERANGE)
                return true;
        if (spi < 0)
                return false;

        slot = &state->stack[spi];

        for (i = 0; i < BPF_REG_SIZE; i++)
                if (slot->slot_type[i] == STACK_IRQ_FLAG)
                        return false;
        return true;
}

static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        struct bpf_func_state *state = func(env, reg);
        struct bpf_stack_state *slot;
        struct bpf_reg_state *st;
        int spi, i;

        spi = irq_flag_get_spi(env, reg);
        if (spi < 0)
                return -EINVAL;

        slot = &state->stack[spi];
        st = &slot->spilled_ptr;

        if (!st->ref_obj_id)
                return -EINVAL;

        for (i = 0; i < BPF_REG_SIZE; i++)
                if (slot->slot_type[i] != STACK_IRQ_FLAG)
                        return -EINVAL;
        return 0;
}

/* Check if given stack slot is "special":
 *   - spilled register state (STACK_SPILL);
 *   - dynptr state (STACK_DYNPTR);
 *   - iter state (STACK_ITER).
 *   - irq flag state (STACK_IRQ_FLAG)
 */
static bool is_stack_slot_special(const struct bpf_stack_state *stack)
{
        enum bpf_stack_slot_type type = stack->slot_type[BPF_REG_SIZE - 1];

        switch (type) {
        case STACK_SPILL:
        case STACK_DYNPTR:
        case STACK_ITER:
        case STACK_IRQ_FLAG:
                return true;
        case STACK_INVALID:
        case STACK_MISC:
        case STACK_ZERO:
                return false;
        default:
                WARN_ONCE(1, "unknown stack slot type %d\n", type);
                return true;
        }
}

/* The reg state of a pointer or a bounded scalar was saved when
 * it was spilled to the stack.
 */
static bool is_spilled_reg(const struct bpf_stack_state *stack)
{
        return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL;
}

static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack)
{
        return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL &&
               stack->spilled_ptr.type == SCALAR_VALUE;
}

static bool is_spilled_scalar_reg64(const struct bpf_stack_state *stack)
{
        return stack->slot_type[0] == STACK_SPILL &&
               stack->spilled_ptr.type == SCALAR_VALUE;
}

/* Mark stack slot as STACK_MISC, unless it is already STACK_INVALID, in which
 * case they are equivalent, or it's STACK_ZERO, in which case we preserve
 * more precise STACK_ZERO.
 * Regardless of allow_ptr_leaks setting (i.e., privileged or unprivileged
 * mode), we won't promote STACK_INVALID to STACK_MISC. In privileged case it is
 * unnecessary as both are considered equivalent when loading data and pruning,
 * in case of unprivileged mode it will be incorrect to allow reads of invalid
 * slots.
 */
static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype)
{
        if (*stype == STACK_ZERO)
                return;
        if (*stype == STACK_INVALID)
                return;
        *stype = STACK_MISC;
}

static void scrub_spilled_slot(u8 *stype)
{
        if (*stype != STACK_INVALID)
                *stype = STACK_MISC;
}

/* copy array src of length n * size bytes to dst. dst is reallocated if it's too
 * small to hold src. This is different from krealloc since we don't want to preserve
 * the contents of dst.
 *
 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could
 * not be allocated.
 */
static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags)
{
        size_t alloc_bytes;
        void *orig = dst;
        size_t bytes;

        if (ZERO_OR_NULL_PTR(src))
                goto out;

        if (unlikely(check_mul_overflow(n, size, &bytes)))
                return NULL;

        alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes));
        dst = krealloc(orig, alloc_bytes, flags);
        if (!dst) {
                kfree(orig);
                return NULL;
        }

        memcpy(dst, src, bytes);
out:
        return dst ? dst : ZERO_SIZE_PTR;
}

/* resize an array from old_n items to new_n items. the array is reallocated if it's too
 * small to hold new_n items. new items are zeroed out if the array grows.
 *
 * Contrary to krealloc_array, does not free arr if new_n is zero.
 */
static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size)
{
        size_t alloc_size;
        void *new_arr;

        if (!new_n || old_n == new_n)
                goto out;

        alloc_size = kmalloc_size_roundup(size_mul(new_n, size));
        new_arr = krealloc(arr, alloc_size, GFP_KERNEL);
        if (!new_arr) {
                kfree(arr);
                return NULL;
        }
        arr = new_arr;

        if (new_n > old_n)
                memset(arr + old_n * size, 0, (new_n - old_n) * size);

out:
        return arr ? arr : ZERO_SIZE_PTR;
}

static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf_verifier_state *src)
{
        dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs,
                               sizeof(struct bpf_reference_state), GFP_KERNEL);
        if (!dst->refs)
                return -ENOMEM;

        dst->acquired_refs = src->acquired_refs;
        dst->active_locks = src->active_locks;
        dst->active_preempt_locks = src->active_preempt_locks;
        dst->active_rcu_lock = src->active_rcu_lock;
        dst->active_irq_id = src->active_irq_id;
        return 0;
}

static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
{
        size_t n = src->allocated_stack / BPF_REG_SIZE;

        dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
                                GFP_KERNEL);
        if (!dst->stack)
                return -ENOMEM;

        dst->allocated_stack = src->allocated_stack;
        return 0;
}

static int resize_reference_state(struct bpf_verifier_state *state, size_t n)
{
        state->refs = realloc_array(state->refs, state->acquired_refs, n,
                                    sizeof(struct bpf_reference_state));
        if (!state->refs)
                return -ENOMEM;

        state->acquired_refs = n;
        return 0;
}

/* Possibly update state->allocated_stack to be at least size bytes. Also
 * possibly update the function's high-water mark in its bpf_subprog_info.
 */
static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)
{
        size_t old_n = state->allocated_stack / BPF_REG_SIZE, n;

        /* The stack size is always a multiple of BPF_REG_SIZE. */
        size = round_up(size, BPF_REG_SIZE);
        n = size / BPF_REG_SIZE;

        if (old_n >= n)
                return 0;

        state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state));
        if (!state->stack)
                return -ENOMEM;

        state->allocated_stack = size;

        /* update known max for given subprogram */
        if (env->subprog_info[state->subprogno].stack_depth < size)
                env->subprog_info[state->subprogno].stack_depth = size;

        return 0;
}

/* Acquire a pointer id from the env and update the state->refs to include
 * this new pointer reference.
 * On success, returns a valid pointer id to associate with the register
 * On failure, returns a negative errno.
 */
static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
{
        struct bpf_verifier_state *state = env->cur_state;
        int new_ofs = state->acquired_refs;
        int err;

        err = resize_reference_state(state, state->acquired_refs + 1);
        if (err)
                return NULL;
        state->refs[new_ofs].insn_idx = insn_idx;

        return &state->refs[new_ofs];
}

static int acquire_reference(struct bpf_verifier_env *env, int insn_idx)
{
        struct bpf_reference_state *s;

        s = acquire_reference_state(env, insn_idx);
        if (!s)
                return -ENOMEM;
        s->type = REF_TYPE_PTR;
        s->id = ++env->id_gen;
        return s->id;
}

static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type,
                              int id, void *ptr)
{
        struct bpf_verifier_state *state = env->cur_state;
        struct bpf_reference_state *s;

        s = acquire_reference_state(env, insn_idx);
        if (!s)
                return -ENOMEM;
        s->type = type;
        s->id = id;
        s->ptr = ptr;

        state->active_locks++;
        return 0;
}

static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx)
{
        struct bpf_verifier_state *state = env->cur_state;
        struct bpf_reference_state *s;

        s = acquire_reference_state(env, insn_idx);
        if (!s)
                return -ENOMEM;
        s->type = REF_TYPE_IRQ;
        s->id = ++env->id_gen;

        state->active_irq_id = s->id;
        return s->id;
}

static void release_reference_state(struct bpf_verifier_state *state, int idx)
{
        int last_idx;
        size_t rem;

        /* IRQ state requires the relative ordering of elements remaining the
         * same, since it relies on the refs array to behave as a stack, so that
         * it can detect out-of-order IRQ restore. Hence use memmove to shift
         * the array instead of swapping the final element into the deleted idx.
         */
        last_idx = state->acquired_refs - 1;
        rem = state->acquired_refs - idx - 1;
        if (last_idx && idx != last_idx)
                memmove(&state->refs[idx], &state->refs[idx + 1], sizeof(*state->refs) * rem);
        memset(&state->refs[last_idx], 0, sizeof(*state->refs));
        state->acquired_refs--;
        return;
}

static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr)
{
        int i;

        for (i = 0; i < state->acquired_refs; i++) {
                if (state->refs[i].type != type)
                        continue;
                if (state->refs[i].id == id && state->refs[i].ptr == ptr) {
                        release_reference_state(state, i);
                        state->active_locks--;
                        return 0;
                }
        }
        return -EINVAL;
}

static int release_irq_state(struct bpf_verifier_state *state, int id)
{
        u32 prev_id = 0;
        int i;

        if (id != state->active_irq_id)
                return -EACCES;

        for (i = 0; i < state->acquired_refs; i++) {
                if (state->refs[i].type != REF_TYPE_IRQ)
                        continue;
                if (state->refs[i].id == id) {
                        release_reference_state(state, i);
                        state->active_irq_id = prev_id;
                        return 0;
                } else {
                        prev_id = state->refs[i].id;
                }
        }
        return -EINVAL;
}

static struct bpf_reference_state *find_lock_state(struct bpf_verifier_state *state, enum ref_state_type type,
                                                   int id, void *ptr)
{
        int i;

        for (i = 0; i < state->acquired_refs; i++) {
                struct bpf_reference_state *s = &state->refs[i];

                if (s->type != type)
                        continue;

                if (s->id == id && s->ptr == ptr)
                        return s;
        }
        return NULL;
}

static void free_func_state(struct bpf_func_state *state)
{
        if (!state)
                return;
        kfree(state->stack);
        kfree(state);
}

static void free_verifier_state(struct bpf_verifier_state *state,
                                bool free_self)
{
        int i;

        for (i = 0; i <= state->curframe; i++) {
                free_func_state(state->frame[i]);
                state->frame[i] = NULL;
        }
        kfree(state->refs);
        if (free_self)
                kfree(state);
}

/* copy verifier state from src to dst growing dst stack space
 * when necessary to accommodate larger src stack
 */
static int copy_func_state(struct bpf_func_state *dst,
                           const struct bpf_func_state *src)
{
        memcpy(dst, src, offsetof(struct bpf_func_state, stack));
        return copy_stack_state(dst, src);
}

static int copy_verifier_state(struct bpf_verifier_state *dst_state,
                               const struct bpf_verifier_state *src)
{
        struct bpf_func_state *dst;
        int i, err;

        /* if dst has more stack frames then src frame, free them, this is also
         * necessary in case of exceptional exits using bpf_throw.
         */
        for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
                free_func_state(dst_state->frame[i]);
                dst_state->frame[i] = NULL;
        }
        err = copy_reference_state(dst_state, src);
        if (err)
                return err;
        dst_state->speculative = src->speculative;
        dst_state->in_sleepable = src->in_sleepable;
        dst_state->curframe = src->curframe;
        dst_state->branches = src->branches;
        dst_state->parent = src->parent;
        dst_state->first_insn_idx = src->first_insn_idx;
        dst_state->last_insn_idx = src->last_insn_idx;
        dst_state->insn_hist_start = src->insn_hist_start;
        dst_state->insn_hist_end = src->insn_hist_end;
        dst_state->dfs_depth = src->dfs_depth;
        dst_state->callback_unroll_depth = src->callback_unroll_depth;
        dst_state->used_as_loop_entry = src->used_as_loop_entry;
        dst_state->may_goto_depth = src->may_goto_depth;
        for (i = 0; i <= src->curframe; i++) {
                dst = dst_state->frame[i];
                if (!dst) {
                        dst = kzalloc(sizeof(*dst), GFP_KERNEL);
                        if (!dst)
                                return -ENOMEM;
                        dst_state->frame[i] = dst;
                }
                err = copy_func_state(dst, src->frame[i]);
                if (err)
                        return err;
        }
        return 0;
}

static u32 state_htab_size(struct bpf_verifier_env *env)
{
        return env->prog->len;
}

static struct bpf_verifier_state_list **explored_state(struct bpf_verifier_env *env, int idx)
{
        struct bpf_verifier_state *cur = env->cur_state;
        struct bpf_func_state *state = cur->frame[cur->curframe];

        return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
}

static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_state *b)
{
        int fr;

        if (a->curframe != b->curframe)
                return false;

        for (fr = a->curframe; fr >= 0; fr--)
                if (a->frame[fr]->callsite != b->frame[fr]->callsite)
                        return false;

        return true;
}

/* Open coded iterators allow back-edges in the state graph in order to
 * check unbounded loops that iterators.
 *
 * In is_state_visited() it is necessary to know if explored states are
 * part of some loops in order to decide whether non-exact states
 * comparison could be used:
 * - non-exact states comparison establishes sub-state relation and uses
 *   read and precision marks to do so, these marks are propagated from
 *   children states and thus are not guaranteed to be final in a loop;
 * - exact states comparison just checks if current and explored states
 *   are identical (and thus form a back-edge).
 *
 * Paper "A New Algorithm for Identifying Loops in Decompilation"
 * by Tao Wei, Jian Mao, Wei Zou and Yu Chen [1] presents a convenient
 * algorithm for loop structure detection and gives an overview of
 * relevant terminology. It also has helpful illustrations.
 *
 * [1] https://api.semanticscholar.org/CorpusID:15784067
 *
 * We use a similar algorithm but because loop nested structure is
 * irrelevant for verifier ours is significantly simpler and resembles
 * strongly connected components algorithm from Sedgewick's textbook.
 *
 * Define topmost loop entry as a first node of the loop traversed in a
 * depth first search starting from initial state. The goal of the loop
 * tracking algorithm is to associate topmost loop entries with states
 * derived from these entries.
 *
 * For each step in the DFS states traversal algorithm needs to identify
 * the following situations:
 *
 *          initial                     initial                   initial
 *            |                           |                         |
 *            V                           V                         V
 *           ...                         ...           .---------> hdr
 *            |                           |            |            |
 *            V                           V            |            V
 *           cur                     .-> succ          |    .------...
 *            |                      |    |            |    |       |
 *            V                      |    V            |    V       V
 *           succ                    '-- cur           |   ...     ...
 *                                                     |    |       |
 *                                                     |    V       V
 *                                                     |   succ <- cur
 *                                                     |    |
 *                                                     |    V
 *                                                     |   ...
 *                                                     |    |
 *                                                     '----'
 *
 *  (A) successor state of cur   (B) successor state of cur or it's entry
 *      not yet traversed            are in current DFS path, thus cur and succ
 *                                   are members of the same outermost loop
 *
 *                      initial                  initial
 *                        |                        |
 *                        V                        V
 *                       ...                      ...
 *                        |                        |
 *                        V                        V
 *                .------...               .------...
 *                |       |                |       |
 *                V       V                V       V
 *           .-> hdr     ...              ...     ...
 *           |    |       |                |       |
 *           |    V       V                V       V
 *           |   succ <- cur              succ <- cur
 *           |    |                        |
 *           |    V                        V
 *           |   ...                      ...
 *           |    |                        |
 *           '----'                       exit
 *
 * (C) successor state of cur is a part of some loop but this loop
 *     does not include cur or successor state is not in a loop at all.
 *
 * Algorithm could be described as the following python code:
 *
 *     traversed = set()   # Set of traversed nodes
 *     entries = {}        # Mapping from node to loop entry
 *     depths = {}         # Depth level assigned to graph node
 *     path = set()        # Current DFS path
 *
 *     # Find outermost loop entry known for n
 *     def get_loop_entry(n):
 *         h = entries.get(n, None)
 *         while h in entries and entries[h] != h:
 *             h = entries[h]
 *         return h
 *
 *     # Update n's loop entry if h's outermost entry comes
 *     # before n's outermost entry in current DFS path.
 *     def update_loop_entry(n, h):
 *         n1 = get_loop_entry(n) or n
 *         h1 = get_loop_entry(h) or h
 *         if h1 in path and depths[h1] <= depths[n1]:
 *             entries[n] = h1
 *
 *     def dfs(n, depth):
 *         traversed.add(n)
 *         path.add(n)
 *         depths[n] = depth
 *         for succ in G.successors(n):
 *             if succ not in traversed:
 *                 # Case A: explore succ and update cur's loop entry
 *                 #         only if succ's entry is in current DFS path.
 *                 dfs(succ, depth + 1)
 *                 h = get_loop_entry(succ)
 *                 update_loop_entry(n, h)
 *             else:
 *                 # Case B or C depending on `h1 in path` check in update_loop_entry().
 *                 update_loop_entry(n, succ)
 *         path.remove(n)
 *
 * To adapt this algorithm for use with verifier:
 * - use st->branch == 0 as a signal that DFS of succ had been finished
 *   and cur's loop entry has to be updated (case A), handle this in
 *   update_branch_counts();
 * - use st->branch > 0 as a signal that st is in the current DFS path;
 * - handle cases B and C in is_state_visited();
 * - update topmost loop entry for intermediate states in get_loop_entry().
 */
static struct bpf_verifier_state *get_loop_entry(struct bpf_verifier_state *st)
{
        struct bpf_verifier_state *topmost = st->loop_entry, *old;

        while (topmost && topmost->loop_entry && topmost != topmost->loop_entry)
                topmost = topmost->loop_entry;
        /* Update loop entries for intermediate states to avoid this
         * traversal in future get_loop_entry() calls.
         */
        while (st && st->loop_entry != topmost) {
                old = st->loop_entry;
                st->loop_entry = topmost;
                st = old;
        }
        return topmost;
}

static void update_loop_entry(struct bpf_verifier_state *cur, struct bpf_verifier_state *hdr)
{
        struct bpf_verifier_state *cur1, *hdr1;

        cur1 = get_loop_entry(cur) ?: cur;
        hdr1 = get_loop_entry(hdr) ?: hdr;
        /* The head1->branches check decides between cases B and C in
         * comment for get_loop_entry(). If hdr1->branches == 0 then
         * head's topmost loop entry is not in current DFS path,
         * hence 'cur' and 'hdr' are not in the same loop and there is
         * no need to update cur->loop_entry.
         */
        if (hdr1->branches && hdr1->dfs_depth <= cur1->dfs_depth) {
                cur->loop_entry = hdr;
                hdr->used_as_loop_entry = true;
        }
}

static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
{
        while (st) {
                u32 br = --st->branches;

                /* br == 0 signals that DFS exploration for 'st' is finished,
                 * thus it is necessary to update parent's loop entry if it
                 * turned out that st is a part of some loop.
                 * This is a part of 'case A' in get_loop_entry() comment.
                 */
                if (br == 0 && st->parent && st->loop_entry)
                        update_loop_entry(st->parent, st->loop_entry);

                /* WARN_ON(br > 1) technically makes sense here,
                 * but see comment in push_stack(), hence:
                 */
                WARN_ONCE((int)br < 0,
                          "BUG update_branch_counts:branches_to_explore=%d\n",
                          br);
                if (br)
                        break;
                st = st->parent;
        }
}

static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
                     int *insn_idx, bool pop_log)
{
        struct bpf_verifier_state *cur = env->cur_state;
        struct bpf_verifier_stack_elem *elem, *head = env->head;
        int err;

        if (env->head == NULL)
                return -ENOENT;

        if (cur) {
                err = copy_verifier_state(cur, &head->st);
                if (err)
                        return err;
        }
        if (pop_log)
                bpf_vlog_reset(&env->log, head->log_pos);
        if (insn_idx)
                *insn_idx = head->insn_idx;
        if (prev_insn_idx)
                *prev_insn_idx = head->prev_insn_idx;
        elem = head->next;
        free_verifier_state(&head->st, false);
        kfree(head);
        env->head = elem;
        env->stack_size--;
        return 0;
}

static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
                                             int insn_idx, int prev_insn_idx,
                                             bool speculative)
{
        struct bpf_verifier_state *cur = env->cur_state;
        struct bpf_verifier_stack_elem *elem;
        int err;

        elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
        if (!elem)
                goto err;

        elem->insn_idx = insn_idx;
        elem->prev_insn_idx = prev_insn_idx;
        elem->next = env->head;
        elem->log_pos = env->log.end_pos;
        env->head = elem;
        env->stack_size++;
        err = copy_verifier_state(&elem->st, cur);
        if (err)
                goto err;
        elem->st.speculative |= speculative;
        if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
                verbose(env, "The sequence of %d jumps is too complex.\n",
                        env->stack_size);
                goto err;
        }
        if (elem->st.parent) {
                ++elem->st.parent->branches;
                /* WARN_ON(branches > 2) technically makes sense here,
                 * but
                 * 1. speculative states will bump 'branches' for non-branch
                 * instructions
                 * 2. is_state_visited() heuristics may decide not to create
                 * a new state for a sequence of branches and all such current
                 * and cloned states will be pointing to a single parent state
                 * which might have large 'branches' count.
                 */
        }
        return &elem->st;
err:
        free_verifier_state(env->cur_state, true);
        env->cur_state = NULL;
        /* pop all elements and return */
        while (!pop_stack(env, NULL, NULL, false));
        return NULL;
}

#define CALLER_SAVED_REGS 6
static const int caller_saved[CALLER_SAVED_REGS] = {
        BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
};

/* This helper doesn't clear reg->id */
static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
{
        reg->var_off = tnum_const(imm);
        reg->smin_value = (s64)imm;
        reg->smax_value = (s64)imm;
        reg->umin_value = imm;
        reg->umax_value = imm;

        reg->s32_min_value = (s32)imm;
        reg->s32_max_value = (s32)imm;
        reg->u32_min_value = (u32)imm;
        reg->u32_max_value = (u32)imm;
}

/* Mark the unknown part of a register (variable offset or scalar value) as
 * known to have the value @imm.
 */
static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
{
        /* Clear off and union(map_ptr, range) */
        memset(((u8 *)reg) + sizeof(reg->type), 0,
               offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
        reg->id = 0;
        reg->ref_obj_id = 0;
        ___mark_reg_known(reg, imm);
}

static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)
{
        reg->var_off = tnum_const_subreg(reg->var_off, imm);
        reg->s32_min_value = (s32)imm;
        reg->s32_max_value = (s32)imm;
        reg->u32_min_value = (u32)imm;
        reg->u32_max_value = (u32)imm;
}

/* Mark the 'variable offset' part of a register as zero.  This should be
 * used only on registers holding a pointer type.
 */
static void __mark_reg_known_zero(struct bpf_reg_state *reg)
{
        __mark_reg_known(reg, 0);
}

static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        __mark_reg_known(reg, 0);
        reg->type = SCALAR_VALUE;
        /* all scalars are assumed imprecise initially (unless unprivileged,
         * in which case everything is forced to be precise)
         */
        reg->precise = !env->bpf_capable;
}

static void mark_reg_known_zero(struct bpf_verifier_env *env,
                                struct bpf_reg_state *regs, u32 regno)
{
        if (WARN_ON(regno >= MAX_BPF_REG)) {
                verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
                /* Something bad happened, let's kill all regs */
                for (regno = 0; regno < MAX_BPF_REG; regno++)
                        __mark_reg_not_init(env, regs + regno);
                return;
        }
        __mark_reg_known_zero(regs + regno);
}

static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type,
                              bool first_slot, int dynptr_id)
{
        /* reg->type has no meaning for STACK_DYNPTR, but when we set reg for
         * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply
         * set it unconditionally as it is ignored for STACK_DYNPTR anyway.
         */
        __mark_reg_known_zero(reg);
        reg->type = CONST_PTR_TO_DYNPTR;
        /* Give each dynptr a unique id to uniquely associate slices to it. */
        reg->id = dynptr_id;
        reg->dynptr.type = type;
        reg->dynptr.first_slot = first_slot;
}

static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
{
        if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
                const struct bpf_map *map = reg->map_ptr;

                if (map->inner_map_meta) {
                        reg->type = CONST_PTR_TO_MAP;
                        reg->map_ptr = map->inner_map_meta;
                        /* transfer reg's id which is unique for every map_lookup_elem
                         * as UID of the inner map.
                         */
                        if (btf_record_has_field(map->inner_map_meta->record, BPF_TIMER))
                                reg->map_uid = reg->id;
                        if (btf_record_has_field(map->inner_map_meta->record, BPF_WORKQUEUE))
                                reg->map_uid = reg->id;
                } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
                        reg->type = PTR_TO_XDP_SOCK;
                } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
                           map->map_type == BPF_MAP_TYPE_SOCKHASH) {
                        reg->type = PTR_TO_SOCKET;
                } else {
                        reg->type = PTR_TO_MAP_VALUE;
                }
                return;
        }

        reg->type &= ~PTR_MAYBE_NULL;
}

static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno,
                                struct btf_field_graph_root *ds_head)
{
        __mark_reg_known_zero(&regs[regno]);
        regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC;
        regs[regno].btf = ds_head->btf;
        regs[regno].btf_id = ds_head->value_btf_id;
        regs[regno].off = ds_head->node_offset;
}

static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
{
        return type_is_pkt_pointer(reg->type);
}

static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
{
        return reg_is_pkt_pointer(reg) ||
               reg->type == PTR_TO_PACKET_END;
}

static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg)
{
        return base_type(reg->type) == PTR_TO_MEM &&
                (reg->type & DYNPTR_TYPE_SKB || reg->type & DYNPTR_TYPE_XDP);
}

/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
                                    enum bpf_reg_type which)
{
        /* The register can already have a range from prior markings.
         * This is fine as long as it hasn't been advanced from its
         * origin.
         */
        return reg->type == which &&
               reg->id == 0 &&
               reg->off == 0 &&
               tnum_equals_const(reg->var_off, 0);
}

/* Reset the min/max bounds of a register */
static void __mark_reg_unbounded(struct bpf_reg_state *reg)
{
        reg->smin_value = S64_MIN;
        reg->smax_value = S64_MAX;
        reg->umin_value = 0;
        reg->umax_value = U64_MAX;

        reg->s32_min_value = S32_MIN;
        reg->s32_max_value = S32_MAX;
        reg->u32_min_value = 0;
        reg->u32_max_value = U32_MAX;
}

static void __mark_reg64_unbounded(struct bpf_reg_state *reg)
{
        reg->smin_value = S64_MIN;
        reg->smax_value = S64_MAX;
        reg->umin_value = 0;
        reg->umax_value = U64_MAX;
}

static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
{
        reg->s32_min_value = S32_MIN;
        reg->s32_max_value = S32_MAX;
        reg->u32_min_value = 0;
        reg->u32_max_value = U32_MAX;
}

static void __update_reg32_bounds(struct bpf_reg_state *reg)
{
        struct tnum var32_off = tnum_subreg(reg->var_off);

        /* min signed is max(sign bit) | min(other bits) */
        reg->s32_min_value = max_t(s32, reg->s32_min_value,
                        var32_off.value | (var32_off.mask & S32_MIN));
        /* max signed is min(sign bit) | max(other bits) */
        reg->s32_max_value = min_t(s32, reg->s32_max_value,
                        var32_off.value | (var32_off.mask & S32_MAX));
        reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value);
        reg->u32_max_value = min(reg->u32_max_value,
                                 (u32)(var32_off.value | var32_off.mask));
}

static void __update_reg64_bounds(struct bpf_reg_state *reg)
{
        /* min signed is max(sign bit) | min(other bits) */
        reg->smin_value = max_t(s64, reg->smin_value,
                                reg->var_off.value | (reg->var_off.mask & S64_MIN));
        /* max signed is min(sign bit) | max(other bits) */
        reg->smax_value = min_t(s64, reg->smax_value,
                                reg->var_off.value | (reg->var_off.mask & S64_MAX));
        reg->umin_value = max(reg->umin_value, reg->var_off.value);
        reg->umax_value = min(reg->umax_value,
                              reg->var_off.value | reg->var_off.mask);
}

static void __update_reg_bounds(struct bpf_reg_state *reg)
{
        __update_reg32_bounds(reg);
        __update_reg64_bounds(reg);
}

/* Uses signed min/max values to inform unsigned, and vice-versa */
static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
{
        /* If upper 32 bits of u64/s64 range don't change, we can use lower 32
         * bits to improve our u32/s32 boundaries.
         *
         * E.g., the case where we have upper 32 bits as zero ([10, 20] in
         * u64) is pretty trivial, it's obvious that in u32 we'll also have
         * [10, 20] range. But this property holds for any 64-bit range as
         * long as upper 32 bits in that entire range of values stay the same.
         *
         * E.g., u64 range [0x10000000A, 0x10000000F] ([4294967306, 4294967311]
         * in decimal) has the same upper 32 bits throughout all the values in
         * that range. As such, lower 32 bits form a valid [0xA, 0xF] ([10, 15])
         * range.
         *
         * Note also, that [0xA, 0xF] is a valid range both in u32 and in s32,
         * following the rules outlined below about u64/s64 correspondence
         * (which equally applies to u32 vs s32 correspondence). In general it
         * depends on actual hexadecimal values of 32-bit range. They can form
         * only valid u32, or only valid s32 ranges in some cases.
         *
         * So we use all these insights to derive bounds for subregisters here.
         */
        if ((reg->umin_value >> 32) == (reg->umax_value >> 32)) {
                /* u64 to u32 casting preserves validity of low 32 bits as
                 * a range, if upper 32 bits are the same
                 */
                reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->umin_value);
                reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->umax_value);

                if ((s32)reg->umin_value <= (s32)reg->umax_value) {
                        reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
                        reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
                }
        }
        if ((reg->smin_value >> 32) == (reg->smax_value >> 32)) {
                /* low 32 bits should form a proper u32 range */
                if ((u32)reg->smin_value <= (u32)reg->smax_value) {
                        reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->smin_value);
                        reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->smax_value);
                }
                /* low 32 bits should form a proper s32 range */
                if ((s32)reg->smin_value <= (s32)reg->smax_value) {
                        reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
                        reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
                }
        }
        /* Special case where upper bits form a small sequence of two
         * sequential numbers (in 32-bit unsigned space, so 0xffffffff to
         * 0x00000000 is also valid), while lower bits form a proper s32 range
         * going from negative numbers to positive numbers. E.g., let's say we
         * have s64 range [-1, 1] ([0xffffffffffffffff, 0x0000000000000001]).
         * Possible s64 values are {-1, 0, 1} ({0xffffffffffffffff,
         * 0x0000000000000000, 0x00000000000001}). Ignoring upper 32 bits,
         * we still get a valid s32 range [-1, 1] ([0xffffffff, 0x00000001]).
         * Note that it doesn't have to be 0xffffffff going to 0x00000000 in
         * upper 32 bits. As a random example, s64 range
         * [0xfffffff0fffffff0; 0xfffffff100000010], forms a valid s32 range
         * [-16, 16] ([0xfffffff0; 0x00000010]) in its 32 bit subregister.
         */
        if ((u32)(reg->umin_value >> 32) + 1 == (u32)(reg->umax_value >> 32) &&
            (s32)reg->umin_value < 0 && (s32)reg->umax_value >= 0) {
                reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
                reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
        }
        if ((u32)(reg->smin_value >> 32) + 1 == (u32)(reg->smax_value >> 32) &&
            (s32)reg->smin_value < 0 && (s32)reg->smax_value >= 0) {
                reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
                reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
        }
        /* if u32 range forms a valid s32 range (due to matching sign bit),
         * try to learn from that
         */
        if ((s32)reg->u32_min_value <= (s32)reg->u32_max_value) {
                reg->s32_min_value = max_t(s32, reg->s32_min_value, reg->u32_min_value);
                reg->s32_max_value = min_t(s32, reg->s32_max_value, reg->u32_max_value);
        }
        /* If we cannot cross the sign boundary, then signed and unsigned bounds
         * are the same, so combine.  This works even in the negative case, e.g.
         * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
         */
        if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) {
                reg->u32_min_value = max_t(u32, reg->s32_min_value, reg->u32_min_value);
                reg->u32_max_value = min_t(u32, reg->s32_max_value, reg->u32_max_value);
        }
}

static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
{
        /* If u64 range forms a valid s64 range (due to matching sign bit),
         * try to learn from that. Let's do a bit of ASCII art to see when
         * this is happening. Let's take u64 range first:
         *
         * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
         * |-------------------------------|--------------------------------|
         *
         * Valid u64 range is formed when umin and umax are anywhere in the
         * range [0, U64_MAX], and umin <= umax. u64 case is simple and
         * straightforward. Let's see how s64 range maps onto the same range
         * of values, annotated below the line for comparison:
         *
         * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
         * |-------------------------------|--------------------------------|
         * 0                        S64_MAX S64_MIN                        -1
         *
         * So s64 values basically start in the middle and they are logically
         * contiguous to the right of it, wrapping around from -1 to 0, and
         * then finishing as S64_MAX (0x7fffffffffffffff) right before
         * S64_MIN. We can try drawing the continuity of u64 vs s64 values
         * more visually as mapped to sign-agnostic range of hex values.
         *
         *  u64 start                                               u64 end
         *  _______________________________________________________________
         * /                                                               \
         * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
         * |-------------------------------|--------------------------------|
         * 0                        S64_MAX S64_MIN                        -1
         *                                / \
         * >------------------------------   ------------------------------->
         * s64 continues...        s64 end   s64 start          s64 "midpoint"
         *
         * What this means is that, in general, we can't always derive
         * something new about u64 from any random s64 range, and vice versa.
         *
         * But we can do that in two particular cases. One is when entire
         * u64/s64 range is *entirely* contained within left half of the above
         * diagram or when it is *entirely* contained in the right half. I.e.:
         *
         * |-------------------------------|--------------------------------|
         *     ^                   ^            ^                 ^
         *     A                   B            C                 D
         *
         * [A, B] and [C, D] are contained entirely in their respective halves
         * and form valid contiguous ranges as both u64 and s64 values. [A, B]
         * will be non-negative both as u64 and s64 (and in fact it will be
         * identical ranges no matter the signedness). [C, D] treated as s64
         * will be a range of negative values, while in u64 it will be
         * non-negative range of values larger than 0x8000000000000000.
         *
         * Now, any other range here can't be represented in both u64 and s64
         * simultaneously. E.g., [A, C], [A, D], [B, C], [B, D] are valid
         * contiguous u64 ranges, but they are discontinuous in s64. [B, C]
         * in s64 would be properly presented as [S64_MIN, C] and [B, S64_MAX],
         * for example. Similarly, valid s64 range [D, A] (going from negative
         * to positive values), would be two separate [D, U64_MAX] and [0, A]
         * ranges as u64. Currently reg_state can't represent two segments per
         * numeric domain, so in such situations we can only derive maximal
         * possible range ([0, U64_MAX] for u64, and [S64_MIN, S64_MAX] for s64).
         *
         * So we use these facts to derive umin/umax from smin/smax and vice
         * versa only if they stay within the same "half". This is equivalent
         * to checking sign bit: lower half will have sign bit as zero, upper
         * half have sign bit 1. Below in code we simplify this by just
         * casting umin/umax as smin/smax and checking if they form valid
         * range, and vice versa. Those are equivalent checks.
         */
        if ((s64)reg->umin_value <= (s64)reg->umax_value) {
                reg->smin_value = max_t(s64, reg->smin_value, reg->umin_value);
                reg->smax_value = min_t(s64, reg->smax_value, reg->umax_value);
        }
        /* If we cannot cross the sign boundary, then signed and unsigned bounds
         * are the same, so combine.  This works even in the negative case, e.g.
         * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
         */
        if ((u64)reg->smin_value <= (u64)reg->smax_value) {
                reg->umin_value = max_t(u64, reg->smin_value, reg->umin_value);
                reg->umax_value = min_t(u64, reg->smax_value, reg->umax_value);
        }
}

static void __reg_deduce_mixed_bounds(struct bpf_reg_state *reg)
{
        /* Try to tighten 64-bit bounds from 32-bit knowledge, using 32-bit
         * values on both sides of 64-bit range in hope to have tighter range.
         * E.g., if r1 is [0x1'00000000, 0x3'80000000], and we learn from
         * 32-bit signed > 0 operation that s32 bounds are now [1; 0x7fffffff].
         * With this, we can substitute 1 as low 32-bits of _low_ 64-bit bound
         * (0x100000000 -> 0x100000001) and 0x7fffffff as low 32-bits of
         * _high_ 64-bit bound (0x380000000 -> 0x37fffffff) and arrive at a
         * better overall bounds for r1 as [0x1'000000001; 0x3'7fffffff].
         * We just need to make sure that derived bounds we are intersecting
         * with are well-formed ranges in respective s64 or u64 domain, just
         * like we do with similar kinds of 32-to-64 or 64-to-32 adjustments.
         */
        __u64 new_umin, new_umax;
        __s64 new_smin, new_smax;

        /* u32 -> u64 tightening, it's always well-formed */
        new_umin = (reg->umin_value & ~0xffffffffULL) | reg->u32_min_value;
        new_umax = (reg->umax_value & ~0xffffffffULL) | reg->u32_max_value;
        reg->umin_value = max_t(u64, reg->umin_value, new_umin);
        reg->umax_value = min_t(u64, reg->umax_value, new_umax);
        /* u32 -> s64 tightening, u32 range embedded into s64 preserves range validity */
        new_smin = (reg->smin_value & ~0xffffffffULL) | reg->u32_min_value;
        new_smax = (reg->smax_value & ~0xffffffffULL) | reg->u32_max_value;
        reg->smin_value = max_t(s64, reg->smin_value, new_smin);
        reg->smax_value = min_t(s64, reg->smax_value, new_smax);

        /* if s32 can be treated as valid u32 range, we can use it as well */
        if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) {
                /* s32 -> u64 tightening */
                new_umin = (reg->umin_value & ~0xffffffffULL) | (u32)reg->s32_min_value;
                new_umax = (reg->umax_value & ~0xffffffffULL) | (u32)reg->s32_max_value;
                reg->umin_value = max_t(u64, reg->umin_value, new_umin);
                reg->umax_value = min_t(u64, reg->umax_value, new_umax);
                /* s32 -> s64 tightening */
                new_smin = (reg->smin_value & ~0xffffffffULL) | (u32)reg->s32_min_value;
                new_smax = (reg->smax_value & ~0xffffffffULL) | (u32)reg->s32_max_value;
                reg->smin_value = max_t(s64, reg->smin_value, new_smin);
                reg->smax_value = min_t(s64, reg->smax_value, new_smax);
        }

        /* Here we would like to handle a special case after sign extending load,
         * when upper bits for a 64-bit range are all 1s or all 0s.
         *
         * Upper bits are all 1s when register is in a range:
         *   [0xffff_ffff_0000_0000, 0xffff_ffff_ffff_ffff]
         * Upper bits are all 0s when register is in a range:
         *   [0x0000_0000_0000_0000, 0x0000_0000_ffff_ffff]
         * Together this forms are continuous range:
         *   [0xffff_ffff_0000_0000, 0x0000_0000_ffff_ffff]
         *
         * Now, suppose that register range is in fact tighter:
         *   [0xffff_ffff_8000_0000, 0x0000_0000_ffff_ffff] (R)
         * Also suppose that it's 32-bit range is positive,
         * meaning that lower 32-bits of the full 64-bit register
         * are in the range:
         *   [0x0000_0000, 0x7fff_ffff] (W)
         *
         * If this happens, then any value in a range:
         *   [0xffff_ffff_0000_0000, 0xffff_ffff_7fff_ffff]
         * is smaller than a lowest bound of the range (R):
         *   0xffff_ffff_8000_0000
         * which means that upper bits of the full 64-bit register
         * can't be all 1s, when lower bits are in range (W).
         *
         * Note that:
         *  - 0xffff_ffff_8000_0000 == (s64)S32_MIN
         *  - 0x0000_0000_7fff_ffff == (s64)S32_MAX
         * These relations are used in the conditions below.
         */
        if (reg->s32_min_value >= 0 && reg->smin_value >= S32_MIN && reg->smax_value <= S32_MAX) {
                reg->smin_value = reg->s32_min_value;
                reg->smax_value = reg->s32_max_value;
                reg->umin_value = reg->s32_min_value;
                reg->umax_value = reg->s32_max_value;
                reg->var_off = tnum_intersect(reg->var_off,
                                              tnum_range(reg->smin_value, reg->smax_value));
        }
}

static void __reg_deduce_bounds(struct bpf_reg_state *reg)
{
        __reg32_deduce_bounds(reg);
        __reg64_deduce_bounds(reg);
        __reg_deduce_mixed_bounds(reg);
}

/* Attempts to improve var_off based on unsigned min/max information */
static void __reg_bound_offset(struct bpf_reg_state *reg)
{
        struct tnum var64_off = tnum_intersect(reg->var_off,
                                               tnum_range(reg->umin_value,
                                                          reg->umax_value));
        struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off),
                                               tnum_range(reg->u32_min_value,
                                                          reg->u32_max_value));

        reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
}

static void reg_bounds_sync(struct bpf_reg_state *reg)
{
        /* We might have learned new bounds from the var_off. */
        __update_reg_bounds(reg);
        /* We might have learned something about the sign bit. */
        __reg_deduce_bounds(reg);
        __reg_deduce_bounds(reg);
        /* We might have learned some bits from the bounds. */
        __reg_bound_offset(reg);
        /* Intersecting with the old var_off might have improved our bounds
         * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
         * then new var_off is (0; 0x7f...fc) which improves our umax.
         */
        __update_reg_bounds(reg);
}

static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
                                   struct bpf_reg_state *reg, const char *ctx)
{
        const char *msg;

        if (reg->umin_value > reg->umax_value ||
            reg->smin_value > reg->smax_value ||
            reg->u32_min_value > reg->u32_max_value ||
            reg->s32_min_value > reg->s32_max_value) {
                    msg = "range bounds violation";
                    goto out;
        }

        if (tnum_is_const(reg->var_off)) {
                u64 uval = reg->var_off.value;
                s64 sval = (s64)uval;

                if (reg->umin_value != uval || reg->umax_value != uval ||
                    reg->smin_value != sval || reg->smax_value != sval) {
                        msg = "const tnum out of sync with range bounds";
                        goto out;
                }
        }

        if (tnum_subreg_is_const(reg->var_off)) {
                u32 uval32 = tnum_subreg(reg->var_off).value;
                s32 sval32 = (s32)uval32;

                if (reg->u32_min_value != uval32 || reg->u32_max_value != uval32 ||
                    reg->s32_min_value != sval32 || reg->s32_max_value != sval32) {
                        msg = "const subreg tnum out of sync with range bounds";
                        goto out;
                }
        }

        return 0;
out:
        verbose(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] "
                "s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)\n",
                ctx, msg, reg->umin_value, reg->umax_value,
                reg->smin_value, reg->smax_value,
                reg->u32_min_value, reg->u32_max_value,
                reg->s32_min_value, reg->s32_max_value,
                reg->var_off.value, reg->var_off.mask);
        if (env->test_reg_invariants)
                return -EFAULT;
        __mark_reg_unbounded(reg);
        return 0;
}

static bool __reg32_bound_s64(s32 a)
{
        return a >= 0 && a <= S32_MAX;
}

static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
{
        reg->umin_value = reg->u32_min_value;
        reg->umax_value = reg->u32_max_value;

        /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must
         * be positive otherwise set to worse case bounds and refine later
         * from tnum.
         */
        if (__reg32_bound_s64(reg->s32_min_value) &&
            __reg32_bound_s64(reg->s32_max_value)) {
                reg->smin_value = reg->s32_min_value;
                reg->smax_value = reg->s32_max_value;
        } else {
                reg->smin_value = 0;
                reg->smax_value = U32_MAX;
        }
}

/* Mark a register as having a completely unknown (scalar) value. */
static void __mark_reg_unknown_imprecise(struct bpf_reg_state *reg)
{
        /*
         * Clear type, off, and union(map_ptr, range) and
         * padding between 'type' and union
         */
        memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
        reg->type = SCALAR_VALUE;
        reg->id = 0;
        reg->ref_obj_id = 0;
        reg->var_off = tnum_unknown;
        reg->frameno = 0;
        reg->precise = false;
        __mark_reg_unbounded(reg);
}

/* Mark a register as having a completely unknown (scalar) value,
 * initialize .precise as true when not bpf capable.
 */
static void __mark_reg_unknown(const struct bpf_verifier_env *env,
                               struct bpf_reg_state *reg)
{
        __mark_reg_unknown_imprecise(reg);
        reg->precise = !env->bpf_capable;
}

static void mark_reg_unknown(struct bpf_verifier_env *env,
                             struct bpf_reg_state *regs, u32 regno)
{
        if (WARN_ON(regno >= MAX_BPF_REG)) {
                verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
                /* Something bad happened, let's kill all regs except FP */
                for (regno = 0; regno < BPF_REG_FP; regno++)
                        __mark_reg_not_init(env, regs + regno);
                return;
        }
        __mark_reg_unknown(env, regs + regno);
}

static int __mark_reg_s32_range(struct bpf_verifier_env *env,
                                struct bpf_reg_state *regs,
                                u32 regno,
                                s32 s32_min,
                                s32 s32_max)
{
        struct bpf_reg_state *reg = regs + regno;

        reg->s32_min_value = max_t(s32, reg->s32_min_value, s32_min);
        reg->s32_max_value = min_t(s32, reg->s32_max_value, s32_max);

        reg->smin_value = max_t(s64, reg->smin_value, s32_min);
        reg->smax_value = min_t(s64, reg->smax_value, s32_max);

        reg_bounds_sync(reg);

        return reg_bounds_sanity_check(env, reg, "s32_range");
}

static void __mark_reg_not_init(const struct bpf_verifier_env *env,
                                struct bpf_reg_state *reg)
{
        __mark_reg_unknown(env, reg);
        reg->type = NOT_INIT;
}

static void mark_reg_not_init(struct bpf_verifier_env *env,
                              struct bpf_reg_state *regs, u32 regno)
{
        if (WARN_ON(regno >= MAX_BPF_REG)) {
                verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
                /* Something bad happened, let's kill all regs except FP */
                for (regno = 0; regno < BPF_REG_FP; regno++)
                        __mark_reg_not_init(env, regs + regno);
                return;
        }
        __mark_reg_not_init(env, regs + regno);
}

static void mark_btf_ld_reg(struct bpf_verifier_env *env,
                            struct bpf_reg_state *regs, u32 regno,
                            enum bpf_reg_type reg_type,
                            struct btf *btf, u32 btf_id,
                            enum bpf_type_flag flag)
{
        if (reg_type == SCALAR_VALUE) {
                mark_reg_unknown(env, regs, regno);
                return;
        }
        mark_reg_known_zero(env, regs, regno);
        regs[regno].type = PTR_TO_BTF_ID | flag;
        regs[regno].btf = btf;
        regs[regno].btf_id = btf_id;
        if (type_may_be_null(flag))
                regs[regno].id = ++env->id_gen;
}

#define DEF_NOT_SUBREG        (0)
static void init_reg_state(struct bpf_verifier_env *env,
                           struct bpf_func_state *state)
{
        struct bpf_reg_state *regs = state->regs;
        int i;

        for (i = 0; i < MAX_BPF_REG; i++) {
                mark_reg_not_init(env, regs, i);
                regs[i].live = REG_LIVE_NONE;
                regs[i].parent = NULL;
                regs[i].subreg_def = DEF_NOT_SUBREG;
        }

        /* frame pointer */
        regs[BPF_REG_FP].type = PTR_TO_STACK;
        mark_reg_known_zero(env, regs, BPF_REG_FP);
        regs[BPF_REG_FP].frameno = state->frameno;
}

static struct bpf_retval_range retval_range(s32 minval, s32 maxval)
{
        return (struct bpf_retval_range){ minval, maxval };
}

#define BPF_MAIN_FUNC (-1)
static void init_func_state(struct bpf_verifier_env *env,
                            struct bpf_func_state *state,
                            int callsite, int frameno, int subprogno)
{
        state->callsite = callsite;
        state->frameno = frameno;
        state->subprogno = subprogno;
        state->callback_ret_range = retval_range(0, 0);
        init_reg_state(env, state);
        mark_verifier_state_scratched(env);
}

/* Similar to push_stack(), but for async callbacks */
static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
                                                int insn_idx, int prev_insn_idx,
                                                int subprog, bool is_sleepable)
{
        struct bpf_verifier_stack_elem *elem;
        struct bpf_func_state *frame;

        elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
        if (!elem)
                goto err;

        elem->insn_idx = insn_idx;
        elem->prev_insn_idx = prev_insn_idx;
        elem->next = env->head;
        elem->log_pos = env->log.end_pos;
        env->head = elem;
        env->stack_size++;
        if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
                verbose(env,
                        "The sequence of %d jumps is too complex for async cb.\n",
                        env->stack_size);
                goto err;
        }
        /* Unlike push_stack() do not copy_verifier_state().
         * The caller state doesn't matter.
         * This is async callback. It starts in a fresh stack.
         * Initialize it similar to do_check_common().
         * But we do need to make sure to not clobber insn_hist, so we keep
         * chaining insn_hist_start/insn_hist_end indices as for a normal
         * child state.
         */
        elem->st.branches = 1;
        elem->st.in_sleepable = is_sleepable;
        elem->st.insn_hist_start = env->cur_state->insn_hist_end;
        elem->st.insn_hist_end = elem->st.insn_hist_start;
        frame = kzalloc(sizeof(*frame), GFP_KERNEL);
        if (!frame)
                goto err;
        init_func_state(env, frame,
                        BPF_MAIN_FUNC /* callsite */,
                        0 /* frameno within this callchain */,
                        subprog /* subprog number within this prog */);
        elem->st.frame[0] = frame;
        return &elem->st;
err:
        free_verifier_state(env->cur_state, true);
        env->cur_state = NULL;
        /* pop all elements and return */
        while (!pop_stack(env, NULL, NULL, false));
        return NULL;
}


enum reg_arg_type {
        SRC_OP,                /* register is used as source operand */
        DST_OP,                /* register is used as destination operand */
        DST_OP_NO_MARK        /* same as above, check only, don't mark */
};

static int cmp_subprogs(const void *a, const void *b)
{
        return ((struct bpf_subprog_info *)a)->start -
               ((struct bpf_subprog_info *)b)->start;
}

/* Find subprogram that contains instruction at 'off' */
static struct bpf_subprog_info *find_containing_subprog(struct bpf_verifier_env *env, int off)
{
        struct bpf_subprog_info *vals = env->subprog_info;
        int l, r, m;

        if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0)
                return NULL;

        l = 0;
        r = env->subprog_cnt - 1;
        while (l < r) {
                m = l + (r - l + 1) / 2;
                if (vals[m].start <= off)
                        l = m;
                else
                        r = m - 1;
        }
        return &vals[l];
}

/* Find subprogram that starts exactly at 'off' */
static int find_subprog(struct bpf_verifier_env *env, int off)
{
        struct bpf_subprog_info *p;

        p = find_containing_subprog(env, off);
        if (!p || p->start != off)
                return -ENOENT;
        return p - env->subprog_info;
}

static int add_subprog(struct bpf_verifier_env *env, int off)
{
        int insn_cnt = env->prog->len;
        int ret;

        if (off >= insn_cnt || off < 0) {
                verbose(env, "call to invalid destination\n");
                return -EINVAL;
        }
        ret = find_subprog(env, off);
        if (ret >= 0)
                return ret;
        if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
                verbose(env, "too many subprograms\n");
                return -E2BIG;
        }
        /* determine subprog starts. The end is one before the next starts */
        env->subprog_info[env->subprog_cnt++].start = off;
        sort(env->subprog_info, env->subprog_cnt,
             sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
        return env->subprog_cnt - 1;
}

static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env)
{
        struct bpf_prog_aux *aux = env->prog->aux;
        struct btf *btf = aux->btf;
        const struct btf_type *t;
        u32 main_btf_id, id;
        const char *name;
        int ret, i;

        /* Non-zero func_info_cnt implies valid btf */
        if (!aux->func_info_cnt)
                return 0;
        main_btf_id = aux->func_info[0].type_id;

        t = btf_type_by_id(btf, main_btf_id);
        if (!t) {
                verbose(env, "invalid btf id for main subprog in func_info\n");
                return -EINVAL;
        }

        name = btf_find_decl_tag_value(btf, t, -1, "exception_callback:");
        if (IS_ERR(name)) {
                ret = PTR_ERR(name);
                /* If there is no tag present, there is no exception callback */
                if (ret == -ENOENT)
                        ret = 0;
                else if (ret == -EEXIST)
                        verbose(env, "multiple exception callback tags for main subprog\n");
                return ret;
        }

        ret = btf_find_by_name_kind(btf, name, BTF_KIND_FUNC);
        if (ret < 0) {
                verbose(env, "exception callback '%s' could not be found in BTF\n", name);
                return ret;
        }
        id = ret;
        t = btf_type_by_id(btf, id);
        if (btf_func_linkage(t) != BTF_FUNC_GLOBAL) {
                verbose(env, "exception callback '%s' must have global linkage\n", name);
                return -EINVAL;
        }
        ret = 0;
        for (i = 0; i < aux->func_info_cnt; i++) {
                if (aux->func_info[i].type_id != id)
                        continue;
                ret = aux->func_info[i].insn_off;
                /* Further func_info and subprog checks will also happen
                 * later, so assume this is the right insn_off for now.
                 */
                if (!ret) {
                        verbose(env, "invalid exception callback insn_off in func_info: 0\n");
                        ret = -EINVAL;
                }
        }
        if (!ret) {
                verbose(env, "exception callback type id not found in func_info\n");
                ret = -EINVAL;
        }
        return ret;
}

#define MAX_KFUNC_DESCS 256
#define MAX_KFUNC_BTFS        256

struct bpf_kfunc_desc {
        struct btf_func_model func_model;
        u32 func_id;
        s32 imm;
        u16 offset;
        unsigned long addr;
};

struct bpf_kfunc_btf {
        struct btf *btf;
        struct module *module;
        u16 offset;
};

struct bpf_kfunc_desc_tab {
        /* Sorted by func_id (BTF ID) and offset (fd_array offset) during
         * verification. JITs do lookups by bpf_insn, where func_id may not be
         * available, therefore at the end of verification do_misc_fixups()
         * sorts this by imm and offset.
         */
        struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
        u32 nr_descs;
};

struct bpf_kfunc_btf_tab {
        struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS];
        u32 nr_descs;
};

static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)
{
        const struct bpf_kfunc_desc *d0 = a;
        const struct bpf_kfunc_desc *d1 = b;

        /* func_id is not greater than BTF_MAX_TYPE */
        return d0->func_id - d1->func_id ?: d0->offset - d1->offset;
}

static int kfunc_btf_cmp_by_off(const void *a, const void *b)
{
        const struct bpf_kfunc_btf *d0 = a;
        const struct bpf_kfunc_btf *d1 = b;

        return d0->offset - d1->offset;
}

static const struct bpf_kfunc_desc *
find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
{
        struct bpf_kfunc_desc desc = {
                .func_id = func_id,
                .offset = offset,
        };
        struct bpf_kfunc_desc_tab *tab;

        tab = prog->aux->kfunc_tab;
        return bsearch(&desc, tab->descs, tab->nr_descs,
                       sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
}

int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
                       u16 btf_fd_idx, u8 **func_addr)
{
        const struct bpf_kfunc_desc *desc;

        desc = find_kfunc_desc(prog, func_id, btf_fd_idx);
        if (!desc)
                return -EFAULT;

        *func_addr = (u8 *)desc->addr;
        return 0;
}

static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
                                         s16 offset)
{
        struct bpf_kfunc_btf kf_btf = { .offset = offset };
        struct bpf_kfunc_btf_tab *tab;
        struct bpf_kfunc_btf *b;
        struct module *mod;
        struct btf *btf;
        int btf_fd;

        tab = env->prog->aux->kfunc_btf_tab;
        b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
                    sizeof(tab->descs[0]), kfunc_btf_cmp_by_off);
        if (!b) {
                if (tab->nr_descs == MAX_KFUNC_BTFS) {
                        verbose(env, "too many different module BTFs\n");
                        return ERR_PTR(-E2BIG);
                }

                if (bpfptr_is_null(env->fd_array)) {
                        verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
                        return ERR_PTR(-EPROTO);
                }

                if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,
                                            offset * sizeof(btf_fd),
                                            sizeof(btf_fd)))
                        return ERR_PTR(-EFAULT);

                btf = btf_get_by_fd(btf_fd);
                if (IS_ERR(btf)) {
                        verbose(env, "invalid module BTF fd specified\n");
                        return btf;
                }

                if (!btf_is_module(btf)) {
                        verbose(env, "BTF fd for kfunc is not a module BTF\n");
                        btf_put(btf);
                        return ERR_PTR(-EINVAL);
                }

                mod = btf_try_get_module(btf);
                if (!mod) {
                        btf_put(btf);
                        return ERR_PTR(-ENXIO);
                }

                b = &tab->descs[tab->nr_descs++];
                b->btf = btf;
                b->module = mod;
                b->offset = offset;

                /* sort() reorders entries by value, so b may no longer point
                 * to the right entry after this
                 */
                sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
                     kfunc_btf_cmp_by_off, NULL);
        } else {
                btf = b->btf;
        }

        return btf;
}

void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab)
{
        if (!tab)
                return;

        while (tab->nr_descs--) {
                module_put(tab->descs[tab->nr_descs].module);
                btf_put(tab->descs[tab->nr_descs].btf);
        }
        kfree(tab);
}

static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)
{
        if (offset) {
                if (offset < 0) {
                        /* In the future, this can be allowed to increase limit
                         * of fd index into fd_array, interpreted as u16.
                         */
                        verbose(env, "negative offset disallowed for kernel module function call\n");
                        return ERR_PTR(-EINVAL);
                }

                return __find_kfunc_desc_btf(env, offset);
        }
        return btf_vmlinux ?: ERR_PTR(-ENOENT);
}

static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
{
        const struct btf_type *func, *func_proto;
        struct bpf_kfunc_btf_tab *btf_tab;
        struct bpf_kfunc_desc_tab *tab;
        struct bpf_prog_aux *prog_aux;
        struct bpf_kfunc_desc *desc;
        const char *func_name;
        struct btf *desc_btf;
        unsigned long call_imm;
        unsigned long addr;
        int err;

        prog_aux = env->prog->aux;
        tab = prog_aux->kfunc_tab;
        btf_tab = prog_aux->kfunc_btf_tab;
        if (!tab) {
                if (!btf_vmlinux) {
                        verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
                        return -ENOTSUPP;
                }

                if (!env->prog->jit_requested) {
                        verbose(env, "JIT is required for calling kernel function\n");
                        return -ENOTSUPP;
                }

                if (!bpf_jit_supports_kfunc_call()) {
                        verbose(env, "JIT does not support calling kernel function\n");
                        return -ENOTSUPP;
                }

                if (!env->prog->gpl_compatible) {
                        verbose(env, "cannot call kernel function from non-GPL compatible program\n");
                        return -EINVAL;
                }

                tab = kzalloc(sizeof(*tab), GFP_KERNEL);
                if (!tab)
                        return -ENOMEM;
                prog_aux->kfunc_tab = tab;
        }

        /* func_id == 0 is always invalid, but instead of returning an error, be
         * conservative and wait until the code elimination pass before returning
         * error, so that invalid calls that get pruned out can be in BPF programs
         * loaded from userspace.  It is also required that offset be untouched
         * for such calls.
         */
        if (!func_id && !offset)
                return 0;

        if (!btf_tab && offset) {
                btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL);
                if (!btf_tab)
                        return -ENOMEM;
                prog_aux->kfunc_btf_tab = btf_tab;
        }

        desc_btf = find_kfunc_desc_btf(env, offset);
        if (IS_ERR(desc_btf)) {
                verbose(env, "failed to find BTF for kernel function\n");
                return PTR_ERR(desc_btf);
        }

        if (find_kfunc_desc(env->prog, func_id, offset))
                return 0;

        if (tab->nr_descs == MAX_KFUNC_DESCS) {
                verbose(env, "too many different kernel function calls\n");
                return -E2BIG;
        }

        func = btf_type_by_id(desc_btf, func_id);
        if (!func || !btf_type_is_func(func)) {
                verbose(env, "kernel btf_id %u is not a function\n",
                        func_id);
                return -EINVAL;
        }
        func_proto = btf_type_by_id(desc_btf, func->type);
        if (!func_proto || !btf_type_is_func_proto(func_proto)) {
                verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
                        func_id);
                return -EINVAL;
        }

        func_name = btf_name_by_offset(desc_btf, func->name_off);
        addr = kallsyms_lookup_name(func_name);
        if (!addr) {
                verbose(env, "cannot find address for kernel function %s\n",
                        func_name);
                return -EINVAL;
        }
        specialize_kfunc(env, func_id, offset, &addr);

        if (bpf_jit_supports_far_kfunc_call()) {
                call_imm = func_id;
        } else {
                call_imm = BPF_CALL_IMM(addr);
                /* Check whether the relative offset overflows desc->imm */
                if ((unsigned long)(s32)call_imm != call_imm) {
                        verbose(env, "address of kernel function %s is out of range\n",
                                func_name);
                        return -EINVAL;
                }
        }

        if (bpf_dev_bound_kfunc_id(func_id)) {
                err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);
                if (err)
                        return err;
        }

        desc = &tab->descs[tab->nr_descs++];
        desc->func_id = func_id;
        desc->imm = call_imm;
        desc->offset = offset;
        desc->addr = addr;
        err = btf_distill_func_proto(&env->log, desc_btf,
                                     func_proto, func_name,
                                     &desc->func_model);
        if (!err)
                sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
                     kfunc_desc_cmp_by_id_off, NULL);
        return err;
}

static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b)
{
        const struct bpf_kfunc_desc *d0 = a;
        const struct bpf_kfunc_desc *d1 = b;

        if (d0->imm != d1->imm)
                return d0->imm < d1->imm ? -1 : 1;
        if (d0->offset != d1->offset)
                return d0->offset < d1->offset ? -1 : 1;
        return 0;
}

static void sort_kfunc_descs_by_imm_off(struct bpf_prog *prog)
{
        struct bpf_kfunc_desc_tab *tab;

        tab = prog->aux->kfunc_tab;
        if (!tab)
                return;

        sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
             kfunc_desc_cmp_by_imm_off, NULL);
}

bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
{
        return !!prog->aux->kfunc_tab;
}

const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
                         const struct bpf_insn *insn)
{
        const struct bpf_kfunc_desc desc = {
                .imm = insn->imm,
                .offset = insn->off,
        };
        const struct bpf_kfunc_desc *res;
        struct bpf_kfunc_desc_tab *tab;

        tab = prog->aux->kfunc_tab;
        res = bsearch(&desc, tab->descs, tab->nr_descs,
                      sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off);

        return res ? &res->func_model : NULL;
}

static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
{
        struct bpf_subprog_info *subprog = env->subprog_info;
        int i, ret, insn_cnt = env->prog->len, ex_cb_insn;
        struct bpf_insn *insn = env->prog->insnsi;

        /* Add entry function. */
        ret = add_subprog(env, 0);
        if (ret)
                return ret;

        for (i = 0; i < insn_cnt; i++, insn++) {
                if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
                    !bpf_pseudo_kfunc_call(insn))
                        continue;

                if (!env->bpf_capable) {
                        verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
                        return -EPERM;
                }

                if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn))
                        ret = add_subprog(env, i + insn->imm + 1);
                else
                        ret = add_kfunc_call(env, insn->imm, insn->off);

                if (ret < 0)
                        return ret;
        }

        ret = bpf_find_exception_callback_insn_off(env);
        if (ret < 0)
                return ret;
        ex_cb_insn = ret;

        /* If ex_cb_insn > 0, this means that the main program has a subprog
         * marked using BTF decl tag to serve as the exception callback.
         */
        if (ex_cb_insn) {
                ret = add_subprog(env, ex_cb_insn);
                if (ret < 0)
                        return ret;
                for (i = 1; i < env->subprog_cnt; i++) {
                        if (env->subprog_info[i].start != ex_cb_insn)
                                continue;
                        env->exception_callback_subprog = i;
                        mark_subprog_exc_cb(env, i);
                        break;
                }
        }

        /* Add a fake 'exit' subprog which could simplify subprog iteration
         * logic. 'subprog_cnt' should not be increased.
         */
        subprog[env->subprog_cnt].start = insn_cnt;

        if (env->log.level & BPF_LOG_LEVEL2)
                for (i = 0; i < env->subprog_cnt; i++)
                        verbose(env, "func#%d @%d\n", i, subprog[i].start);

        return 0;
}

static int check_subprogs(struct bpf_verifier_env *env)
{
        int i, subprog_start, subprog_end, off, cur_subprog = 0;
        struct bpf_subprog_info *subprog = env->subprog_info;
        struct bpf_insn *insn = env->prog->insnsi;
        int insn_cnt = env->prog->len;

        /* now check that all jumps are within the same subprog */
        subprog_start = subprog[cur_subprog].start;
        subprog_end = subprog[cur_subprog + 1].start;
        for (i = 0; i < insn_cnt; i++) {
                u8 code = insn[i].code;

                if (code == (BPF_JMP | BPF_CALL) &&
                    insn[i].src_reg == 0 &&
                    insn[i].imm == BPF_FUNC_tail_call) {
                        subprog[cur_subprog].has_tail_call = true;
                        subprog[cur_subprog].tail_call_reachable = true;
                }
                if (BPF_CLASS(code) == BPF_LD &&
                    (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND))
                        subprog[cur_subprog].has_ld_abs = true;
                if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
                        goto next;
                if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
                        goto next;
                if (code == (BPF_JMP32 | BPF_JA))
                        off = i + insn[i].imm + 1;
                else
                        off = i + insn[i].off + 1;
                if (off < subprog_start || off >= subprog_end) {
                        verbose(env, "jump out of range from insn %d to %d\n", i, off);
                        return -EINVAL;
                }
next:
                if (i == subprog_end - 1) {
                        /* to avoid fall-through from one subprog into another
                         * the last insn of the subprog should be either exit
                         * or unconditional jump back or bpf_throw call
                         */
                        if (code != (BPF_JMP | BPF_EXIT) &&
                            code != (BPF_JMP32 | BPF_JA) &&
                            code != (BPF_JMP | BPF_JA)) {
                                verbose(env, "last insn is not an exit or jmp\n");
                                return -EINVAL;
                        }
                        subprog_start = subprog_end;
                        cur_subprog++;
                        if (cur_subprog < env->subprog_cnt)
                                subprog_end = subprog[cur_subprog + 1].start;
                }
        }
        return 0;
}

/* Parentage chain of this register (or stack slot) should take care of all
 * issues like callee-saved registers, stack slot allocation time, etc.
 */
static int mark_reg_read(struct bpf_verifier_env *env,
                         const struct bpf_reg_state *state,
                         struct bpf_reg_state *parent, u8 flag)
{
        bool writes = parent == state->parent; /* Observe write marks */
        int cnt = 0;

        while (parent) {
                /* if read wasn't screened by an earlier write ... */
                if (writes && state->live & REG_LIVE_WRITTEN)
                        break;
                if (parent->live & REG_LIVE_DONE) {
                        verbose(env, "verifier BUG type %s var_off %lld off %d\n",
                                reg_type_str(env, parent->type),
                                parent->var_off.value, parent->off);
                        return -EFAULT;
                }
                /* The first condition is more likely to be true than the
                 * second, checked it first.
                 */
                if ((parent->live & REG_LIVE_READ) == flag ||
                    parent->live & REG_LIVE_READ64)
                        /* The parentage chain never changes and
                         * this parent was already marked as LIVE_READ.
                         * There is no need to keep walking the chain again and
                         * keep re-marking all parents as LIVE_READ.
                         * This case happens when the same register is read
                         * multiple times without writes into it in-between.
                         * Also, if parent has the stronger REG_LIVE_READ64 set,
                         * then no need to set the weak REG_LIVE_READ32.
                         */
                        break;
                /* ... then we depend on parent's value */
                parent->live |= flag;
                /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */
                if (flag == REG_LIVE_READ64)
                        parent->live &= ~REG_LIVE_READ32;
                state = parent;
                parent = state->parent;
                writes = true;
                cnt++;
        }

        if (env->longest_mark_read_walk < cnt)
                env->longest_mark_read_walk = cnt;
        return 0;
}

static int mark_stack_slot_obj_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
                                    int spi, int nr_slots)
{
        struct bpf_func_state *state = func(env, reg);
        int err, i;

        for (i = 0; i < nr_slots; i++) {
                struct bpf_reg_state *st = &state->stack[spi - i].spilled_ptr;

                err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64);
                if (err)
                        return err;

                mark_stack_slot_scratched(env, spi - i);
        }
        return 0;
}

static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        int spi;

        /* For CONST_PTR_TO_DYNPTR, it must have already been done by
         * check_reg_arg in check_helper_call and mark_btf_func_reg_size in
         * check_kfunc_call.
         */
        if (reg->type == CONST_PTR_TO_DYNPTR)
                return 0;
        spi = dynptr_get_spi(env, reg);
        if (spi < 0)
                return spi;
        /* Caller ensures dynptr is valid and initialized, which means spi is in
         * bounds and spi is the first dynptr slot. Simply mark stack slot as
         * read.
         */
        return mark_stack_slot_obj_read(env, reg, spi, BPF_DYNPTR_NR_SLOTS);
}

static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
                          int spi, int nr_slots)
{
        return mark_stack_slot_obj_read(env, reg, spi, nr_slots);
}

static int mark_irq_flag_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        int spi;

        spi = irq_flag_get_spi(env, reg);
        if (spi < 0)
                return spi;
        return mark_stack_slot_obj_read(env, reg, spi, 1);
}

/* This function is supposed to be used by the following 32-bit optimization
 * code only. It returns TRUE if the source or destination register operates
 * on 64-bit, otherwise return FALSE.
 */
static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,
                     u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)
{
        u8 code, class, op;

        code = insn->code;
        class = BPF_CLASS(code);
        op = BPF_OP(code);
        if (class == BPF_JMP) {
                /* BPF_EXIT for "main" will reach here. Return TRUE
                 * conservatively.
                 */
                if (op == BPF_EXIT)
                        return true;
                if (op == BPF_CALL) {
                        /* BPF to BPF call will reach here because of marking
                         * caller saved clobber with DST_OP_NO_MARK for which we
                         * don't care the register def because they are anyway
                         * marked as NOT_INIT already.
                         */
                        if (insn->src_reg == BPF_PSEUDO_CALL)
                                return false;
                        /* Helper call will reach here because of arg type
                         * check, conservatively return TRUE.
                         */
                        if (t == SRC_OP)
                                return true;

                        return false;
                }
        }

        if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32))
                return false;

        if (class == BPF_ALU64 || class == BPF_JMP ||
            (class == BPF_ALU && op == BPF_END && insn->imm == 64))
                return true;

        if (class == BPF_ALU || class == BPF_JMP32)
                return false;

        if (class == BPF_LDX) {
                if (t != SRC_OP)
                        return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX;
                /* LDX source must be ptr. */
                return true;
        }

        if (class == BPF_STX) {
                /* BPF_STX (including atomic variants) has multiple source
                 * operands, one of which is a ptr. Check whether the caller is
                 * asking about it.
                 */
                if (t == SRC_OP && reg->type != SCALAR_VALUE)
                        return true;
                return BPF_SIZE(code) == BPF_DW;
        }

        if (class == BPF_LD) {
                u8 mode = BPF_MODE(code);

                /* LD_IMM64 */
                if (mode == BPF_IMM)
                        return true;

                /* Both LD_IND and LD_ABS return 32-bit data. */
                if (t != SRC_OP)
                        return  false;

                /* Implicit ctx ptr. */
                if (regno == BPF_REG_6)
                        return true;

                /* Explicit source could be any width. */
                return true;
        }

        if (class == BPF_ST)
                /* The only source register for BPF_ST is a ptr. */
                return true;

        /* Conservatively return true at default. */
        return true;
}

/* Return the regno defined by the insn, or -1. */
static int insn_def_regno(const struct bpf_insn *insn)
{
        switch (BPF_CLASS(insn->code)) {
        case BPF_JMP:
        case BPF_JMP32:
        case BPF_ST:
                return -1;
        case BPF_STX:
                if ((BPF_MODE(insn->code) == BPF_ATOMIC ||
                     BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) &&
                    (insn->imm & BPF_FETCH)) {
                        if (insn->imm == BPF_CMPXCHG)
                                return BPF_REG_0;
                        else
                                return insn->src_reg;
                } else {
                        return -1;
                }
        default:
                return insn->dst_reg;
        }
}

/* Return TRUE if INSN has defined any 32-bit value explicitly. */
static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)
{
        int dst_reg = insn_def_regno(insn);

        if (dst_reg == -1)
                return false;

        return !is_reg64(env, insn, dst_reg, NULL, DST_OP);
}

static void mark_insn_zext(struct bpf_verifier_env *env,
                           struct bpf_reg_state *reg)
{
        s32 def_idx = reg->subreg_def;

        if (def_idx == DEF_NOT_SUBREG)
                return;

        env->insn_aux_data[def_idx - 1].zext_dst = true;
        /* The dst will be zero extended, so won't be sub-register anymore. */
        reg->subreg_def = DEF_NOT_SUBREG;
}

static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,
                           enum reg_arg_type t)
{
        struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
        struct bpf_reg_state *reg;
        bool rw64;

        if (regno >= MAX_BPF_REG) {
                verbose(env, "R%d is invalid\n", regno);
                return -EINVAL;
        }

        mark_reg_scratched(env, regno);

        reg = &regs[regno];
        rw64 = is_reg64(env, insn, regno, reg, t);
        if (t == SRC_OP) {
                /* check whether register used as source operand can be read */
                if (reg->type == NOT_INIT) {
                        verbose(env, "R%d !read_ok\n", regno);
                        return -EACCES;
                }
                /* We don't need to worry about FP liveness because it's read-only */
                if (regno == BPF_REG_FP)
                        return 0;

                if (rw64)
                        mark_insn_zext(env, reg);

                return mark_reg_read(env, reg, reg->parent,
                                     rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32);
        } else {
                /* check whether register used as dest operand can be written to */
                if (regno == BPF_REG_FP) {
                        verbose(env, "frame pointer is read only\n");
                        return -EACCES;
                }
                reg->live |= REG_LIVE_WRITTEN;
                reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;
                if (t == DST_OP)
                        mark_reg_unknown(env, regs, regno);
        }
        return 0;
}

static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
                         enum reg_arg_type t)
{
        struct bpf_verifier_state *vstate = env->cur_state;
        struct bpf_func_state *state = vstate->frame[vstate->curframe];

        return __check_reg_arg(env, state->regs, regno, t);
}

static int insn_stack_access_flags(int frameno, int spi)
{
        return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;
}

static int insn_stack_access_spi(int insn_flags)
{
        return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK;
}

static int insn_stack_access_frameno(int insn_flags)
{
        return insn_flags & INSN_F_FRAMENO_MASK;
}

static void mark_jmp_point(struct bpf_verifier_env *env, int idx)
{
        env->insn_aux_data[idx].jmp_point = true;
}

static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx)
{
        return env->insn_aux_data[insn_idx].jmp_point;
}

#define LR_FRAMENO_BITS        3
#define LR_SPI_BITS        6
#define LR_ENTRY_BITS        (LR_SPI_BITS + LR_FRAMENO_BITS + 1)
#define LR_SIZE_BITS        4
#define LR_FRAMENO_MASK        ((1ull << LR_FRAMENO_BITS) - 1)
#define LR_SPI_MASK        ((1ull << LR_SPI_BITS)     - 1)
#define LR_SIZE_MASK        ((1ull << LR_SIZE_BITS)    - 1)
#define LR_SPI_OFF        LR_FRAMENO_BITS
#define LR_IS_REG_OFF        (LR_SPI_BITS + LR_FRAMENO_BITS)
#define LINKED_REGS_MAX        6

struct linked_reg {
        u8 frameno;
        union {
                u8 spi;
                u8 regno;
        };
        bool is_reg;
};

struct linked_regs {
        int cnt;
        struct linked_reg entries[LINKED_REGS_MAX];
};

static struct linked_reg *linked_regs_push(struct linked_regs *s)
{
        if (s->cnt < LINKED_REGS_MAX)
                return &s->entries[s->cnt++];

        return NULL;
}

/* Use u64 as a vector of 6 10-bit values, use first 4-bits to track
 * number of elements currently in stack.
 * Pack one history entry for linked registers as 10 bits in the following format:
 * - 3-bits frameno
 * - 6-bits spi_or_reg
 * - 1-bit  is_reg
 */
static u64 linked_regs_pack(struct linked_regs *s)
{
        u64 val = 0;
        int i;

        for (i = 0; i < s->cnt; ++i) {
                struct linked_reg *e = &s->entries[i];
                u64 tmp = 0;

                tmp |= e->frameno;
                tmp |= e->spi << LR_SPI_OFF;
                tmp |= (e->is_reg ? 1 : 0) << LR_IS_REG_OFF;

                val <<= LR_ENTRY_BITS;
                val |= tmp;
        }
        val <<= LR_SIZE_BITS;
        val |= s->cnt;
        return val;
}

static void linked_regs_unpack(u64 val, struct linked_regs *s)
{
        int i;

        s->cnt = val & LR_SIZE_MASK;
        val >>= LR_SIZE_BITS;

        for (i = 0; i < s->cnt; ++i) {
                struct linked_reg *e = &s->entries[i];

                e->frameno =  val & LR_FRAMENO_MASK;
                e->spi     = (val >> LR_SPI_OFF) & LR_SPI_MASK;
                e->is_reg  = (val >> LR_IS_REG_OFF) & 0x1;
                val >>= LR_ENTRY_BITS;
        }
}

/* for any branch, call, exit record the history of jmps in the given state */
static int push_insn_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
                             int insn_flags, u64 linked_regs)
{
        struct bpf_insn_hist_entry *p;
        size_t alloc_size;

        /* combine instruction flags if we already recorded this instruction */
        if (env->cur_hist_ent) {
                /* atomic instructions push insn_flags twice, for READ and
                 * WRITE sides, but they should agree on stack slot
                 */
                WARN_ONCE((env->cur_hist_ent->flags & insn_flags) &&
                          (env->cur_hist_ent->flags & insn_flags) != insn_flags,
                          "verifier insn history bug: insn_idx %d cur flags %x new flags %x\n",
                          env->insn_idx, env->cur_hist_ent->flags, insn_flags);
                env->cur_hist_ent->flags |= insn_flags;
                WARN_ONCE(env->cur_hist_ent->linked_regs != 0,
                          "verifier insn history bug: insn_idx %d linked_regs != 0: %#llx\n",
                          env->insn_idx, env->cur_hist_ent->linked_regs);
                env->cur_hist_ent->linked_regs = linked_regs;
                return 0;
        }

        if (cur->insn_hist_end + 1 > env->insn_hist_cap) {
                alloc_size = size_mul(cur->insn_hist_end + 1, sizeof(*p));
                p = kvrealloc(env->insn_hist, alloc_size, GFP_USER);
                if (!p)
                        return -ENOMEM;
                env->insn_hist = p;
                env->insn_hist_cap = alloc_size / sizeof(*p);
        }

        p = &env->insn_hist[cur->insn_hist_end];
        p->idx = env->insn_idx;
        p->prev_idx = env->prev_insn_idx;
        p->flags = insn_flags;
        p->linked_regs = linked_regs;

        cur->insn_hist_end++;
        env->cur_hist_ent = p;

        return 0;
}

static struct bpf_insn_hist_entry *get_insn_hist_entry(struct bpf_verifier_env *env,
                                                       u32 hist_start, u32 hist_end, int insn_idx)
{
        if (hist_end > hist_start && env->insn_hist[hist_end - 1].idx == insn_idx)
                return &env->insn_hist[hist_end - 1];
        return NULL;
}

/* Backtrack one insn at a time. If idx is not at the top of recorded
 * history then previous instruction came from straight line execution.
 * Return -ENOENT if we exhausted all instructions within given state.
 *
 * It's legal to have a bit of a looping with the same starting and ending
 * insn index within the same state, e.g.: 3->4->5->3, so just because current
 * instruction index is the same as state's first_idx doesn't mean we are
 * done. If there is still some jump history left, we should keep going. We
 * need to take into account that we might have a jump history between given
 * state's parent and itself, due to checkpointing. In this case, we'll have
 * history entry recording a jump from last instruction of parent state and
 * first instruction of given state.
 */
static int get_prev_insn_idx(const struct bpf_verifier_env *env,
                             struct bpf_verifier_state *st,
                             int insn_idx, u32 hist_start, u32 *hist_endp)
{
        u32 hist_end = *hist_endp;
        u32 cnt = hist_end - hist_start;

        if (insn_idx == st->first_insn_idx) {
                if (cnt == 0)
                        return -ENOENT;
                if (cnt == 1 && env->insn_hist[hist_start].idx == insn_idx)
                        return -ENOENT;
        }

        if (cnt && env->insn_hist[hist_end - 1].idx == insn_idx) {
                (*hist_endp)--;
                return env->insn_hist[hist_end - 1].prev_idx;
        } else {
                return insn_idx - 1;
        }
}

static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
{
        const struct btf_type *func;
        struct btf *desc_btf;

        if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
                return NULL;

        desc_btf = find_kfunc_desc_btf(data, insn->off);
        if (IS_ERR(desc_btf))
                return "<error>";

        func = btf_type_by_id(desc_btf, insn->imm);
        return btf_name_by_offset(desc_btf, func->name_off);
}

static inline void bt_init(struct backtrack_state *bt, u32 frame)
{
        bt->frame = frame;
}

static inline void bt_reset(struct backtrack_state *bt)
{
        struct bpf_verifier_env *env = bt->env;

        memset(bt, 0, sizeof(*bt));
        bt->env = env;
}

static inline u32 bt_empty(struct backtrack_state *bt)
{
        u64 mask = 0;
        int i;

        for (i = 0; i <= bt->frame; i++)
                mask |= bt->reg_masks[i] | bt->stack_masks[i];

        return mask == 0;
}

static inline int bt_subprog_enter(struct backtrack_state *bt)
{
        if (bt->frame == MAX_CALL_FRAMES - 1) {
                verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame);
                WARN_ONCE(1, "verifier backtracking bug");
                return -EFAULT;
        }
        bt->frame++;
        return 0;
}

static inline int bt_subprog_exit(struct backtrack_state *bt)
{
        if (bt->frame == 0) {
                verbose(bt->env, "BUG subprog exit from frame 0\n");
                WARN_ONCE(1, "verifier backtracking bug");
                return -EFAULT;
        }
        bt->frame--;
        return 0;
}

static inline void bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
{
        bt->reg_masks[frame] |= 1 << reg;
}

static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
{
        bt->reg_masks[frame] &= ~(1 << reg);
}

static inline void bt_set_reg(struct backtrack_state *bt, u32 reg)
{
        bt_set_frame_reg(bt, bt->frame, reg);
}

static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg)
{
        bt_clear_frame_reg(bt, bt->frame, reg);
}

static inline void bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
{
        bt->stack_masks[frame] |= 1ull << slot;
}

static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
{
        bt->stack_masks[frame] &= ~(1ull << slot);
}

static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame)
{
        return bt->reg_masks[frame];
}

static inline u32 bt_reg_mask(struct backtrack_state *bt)
{
        return bt->reg_masks[bt->frame];
}

static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
{
        return bt->stack_masks[frame];
}

static inline u64 bt_stack_mask(struct backtrack_state *bt)
{
        return bt->stack_masks[bt->frame];
}

static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
{
        return bt->reg_masks[bt->frame] & (1 << reg);
}

static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg)
{
        return bt->reg_masks[frame] & (1 << reg);
}

static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)
{
        return bt->stack_masks[frame] & (1ull << slot);
}

/* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */
static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask)
{
        DECLARE_BITMAP(mask, 64);
        bool first = true;
        int i, n;

        buf[0] = '\0';

        bitmap_from_u64(mask, reg_mask);
        for_each_set_bit(i, mask, 32) {
                n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i);
                first = false;
                buf += n;
                buf_sz -= n;
                if (buf_sz < 0)
                        break;
        }
}
/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */
static void fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
{
        DECLARE_BITMAP(mask, 64);
        bool first = true;
        int i, n;

        buf[0] = '\0';

        bitmap_from_u64(mask, stack_mask);
        for_each_set_bit(i, mask, 64) {
                n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8);
                first = false;
                buf += n;
                buf_sz -= n;
                if (buf_sz < 0)
                        break;
        }
}

/* If any register R in hist->linked_regs is marked as precise in bt,
 * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs.
 */
static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_insn_hist_entry *hist)
{
        struct linked_regs linked_regs;
        bool some_precise = false;
        int i;

        if (!hist || hist->linked_regs == 0)
                return;

        linked_regs_unpack(hist->linked_regs, &linked_regs);
        for (i = 0; i < linked_regs.cnt; ++i) {
                struct linked_reg *e = &linked_regs.entries[i];

                if ((e->is_reg && bt_is_frame_reg_set(bt, e->frameno, e->regno)) ||
                    (!e->is_reg && bt_is_frame_slot_set(bt, e->frameno, e->spi))) {
                        some_precise = true;
                        break;
                }
        }

        if (!some_precise)
                return;

        for (i = 0; i < linked_regs.cnt; ++i) {
                struct linked_reg *e = &linked_regs.entries[i];

                if (e->is_reg)
                        bt_set_frame_reg(bt, e->frameno, e->regno);
                else
                        bt_set_frame_slot(bt, e->frameno, e->spi);
        }
}

static bool calls_callback(struct bpf_verifier_env *env, int insn_idx);

/* For given verifier state backtrack_insn() is called from the last insn to
 * the first insn. Its purpose is to compute a bitmask of registers and
 * stack slots that needs precision in the parent verifier state.
 *
 * @idx is an index of the instruction we are currently processing;
 * @subseq_idx is an index of the subsequent instruction that:
 *   - *would be* executed next, if jump history is viewed in forward order;
 *   - *was* processed previously during backtracking.
 */
static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
                          struct bpf_insn_hist_entry *hist, struct backtrack_state *bt)
{
        const struct bpf_insn_cbs cbs = {
                .cb_call        = disasm_kfunc_name,
                .cb_print        = verbose,
                .private_data        = env,
        };
        struct bpf_insn *insn = env->prog->insnsi + idx;
        u8 class = BPF_CLASS(insn->code);
        u8 opcode = BPF_OP(insn->code);
        u8 mode = BPF_MODE(insn->code);
        u32 dreg = insn->dst_reg;
        u32 sreg = insn->src_reg;
        u32 spi, i, fr;

        if (insn->code == 0)
                return 0;
        if (env->log.level & BPF_LOG_LEVEL2) {
                fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));
                verbose(env, "mark_precise: frame%d: regs=%s ",
                        bt->frame, env->tmp_str_buf);
                fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
                verbose(env, "stack=%s before ", env->tmp_str_buf);
                verbose(env, "%d: ", idx);
                print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
        }

        /* If there is a history record that some registers gained range at this insn,
         * propagate precision marks to those registers, so that bt_is_reg_set()
         * accounts for these registers.
         */
        bt_sync_linked_regs(bt, hist);

        if (class == BPF_ALU || class == BPF_ALU64) {
                if (!bt_is_reg_set(bt, dreg))
                        return 0;
                if (opcode == BPF_END || opcode == BPF_NEG) {
                        /* sreg is reserved and unused
                         * dreg still need precision before this insn
                         */
                        return 0;
                } else if (opcode == BPF_MOV) {
                        if (BPF_SRC(insn->code) == BPF_X) {
                                /* dreg = sreg or dreg = (s8, s16, s32)sreg
                                 * dreg needs precision after this insn
                                 * sreg needs precision before this insn
                                 */
                                bt_clear_reg(bt, dreg);
                                if (sreg != BPF_REG_FP)
                                        bt_set_reg(bt, sreg);
                        } else {
                                /* dreg = K
                                 * dreg needs precision after this insn.
                                 * Corresponding register is already marked
                                 * as precise=true in this verifier state.
                                 * No further markings in parent are necessary
                                 */
                                bt_clear_reg(bt, dreg);
                        }
                } else {
                        if (BPF_SRC(insn->code) == BPF_X) {
                                /* dreg += sreg
                                 * both dreg and sreg need precision
                                 * before this insn
                                 */
                                if (sreg != BPF_REG_FP)
                                        bt_set_reg(bt, sreg);
                        } /* else dreg += K
                           * dreg still needs precision before this insn
                           */
                }
        } else if (class == BPF_LDX) {
                if (!bt_is_reg_set(bt, dreg))
                        return 0;
                bt_clear_reg(bt, dreg);

                /* scalars can only be spilled into stack w/o losing precision.
                 * Load from any other memory can be zero extended.
                 * The desire to keep that precision is already indicated
                 * by 'precise' mark in corresponding register of this state.
                 * No further tracking necessary.
                 */
                if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
                        return 0;
                /* dreg = *(u64 *)[fp - off] was a fill from the stack.
                 * that [fp - off] slot contains scalar that needs to be
                 * tracked with precision
                 */
                spi = insn_stack_access_spi(hist->flags);
                fr = insn_stack_access_frameno(hist->flags);
                bt_set_frame_slot(bt, fr, spi);
        } else if (class == BPF_STX || class == BPF_ST) {
                if (bt_is_reg_set(bt, dreg))
                        /* stx & st shouldn't be using _scalar_ dst_reg
                         * to access memory. It means backtracking
                         * encountered a case of pointer subtraction.
                         */
                        return -ENOTSUPP;
                /* scalars can only be spilled into stack */
                if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
                        return 0;
                spi = insn_stack_access_spi(hist->flags);
                fr = insn_stack_access_frameno(hist->flags);
                if (!bt_is_frame_slot_set(bt, fr, spi))
                        return 0;
                bt_clear_frame_slot(bt, fr, spi);
                if (class == BPF_STX)
                        bt_set_reg(bt, sreg);
        } else if (class == BPF_JMP || class == BPF_JMP32) {
                if (bpf_pseudo_call(insn)) {
                        int subprog_insn_idx, subprog;

                        subprog_insn_idx = idx + insn->imm + 1;
                        subprog = find_subprog(env, subprog_insn_idx);
                        if (subprog < 0)
                                return -EFAULT;

                        if (subprog_is_global(env, subprog)) {
                                /* check that jump history doesn't have any
                                 * extra instructions from subprog; the next
                                 * instruction after call to global subprog
                                 * should be literally next instruction in
                                 * caller program
                                 */
                                WARN_ONCE(idx + 1 != subseq_idx, "verifier backtracking bug");
                                /* r1-r5 are invalidated after subprog call,
                                 * so for global func call it shouldn't be set
                                 * anymore
                                 */
                                if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
                                        verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
                                        WARN_ONCE(1, "verifier backtracking bug");
                                        return -EFAULT;
                                }
                                /* global subprog always sets R0 */
                                bt_clear_reg(bt, BPF_REG_0);
                                return 0;
                        } else {
                                /* static subprog call instruction, which
                                 * means that we are exiting current subprog,
                                 * so only r1-r5 could be still requested as
                                 * precise, r0 and r6-r10 or any stack slot in
                                 * the current frame should be zero by now
                                 */
                                if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
                                        verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
                                        WARN_ONCE(1, "verifier backtracking bug");
                                        return -EFAULT;
                                }
                                /* we are now tracking register spills correctly,
                                 * so any instance of leftover slots is a bug
                                 */
                                if (bt_stack_mask(bt) != 0) {
                                        verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt));
                                        WARN_ONCE(1, "verifier backtracking bug (subprog leftover stack slots)");
                                        return -EFAULT;
                                }
                                /* propagate r1-r5 to the caller */
                                for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
                                        if (bt_is_reg_set(bt, i)) {
                                                bt_clear_reg(bt, i);
                                                bt_set_frame_reg(bt, bt->frame - 1, i);
                                        }
                                }
                                if (bt_subprog_exit(bt))
                                        return -EFAULT;
                                return 0;
                        }
                } else if (is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) {
                        /* exit from callback subprog to callback-calling helper or
                         * kfunc call. Use idx/subseq_idx check to discern it from
                         * straight line code backtracking.
                         * Unlike the subprog call handling above, we shouldn't
                         * propagate precision of r1-r5 (if any requested), as they are
                         * not actually arguments passed directly to callback subprogs
                         */
                        if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
                                verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
                                WARN_ONCE(1, "verifier backtracking bug");
                                return -EFAULT;
                        }
                        if (bt_stack_mask(bt) != 0) {
                                verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt));
                                WARN_ONCE(1, "verifier backtracking bug (callback leftover stack slots)");
                                return -EFAULT;
                        }
                        /* clear r1-r5 in callback subprog's mask */
                        for (i = BPF_REG_1; i <= BPF_REG_5; i++)
                                bt_clear_reg(bt, i);
                        if (bt_subprog_exit(bt))
                                return -EFAULT;
                        return 0;
                } else if (opcode == BPF_CALL) {
                        /* kfunc with imm==0 is invalid and fixup_kfunc_call will
                         * catch this error later. Make backtracking conservative
                         * with ENOTSUPP.
                         */
                        if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0)
                                return -ENOTSUPP;
                        /* regular helper call sets R0 */
                        bt_clear_reg(bt, BPF_REG_0);
                        if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
                                /* if backtracing was looking for registers R1-R5
                                 * they should have been found already.
                                 */
                                verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
                                WARN_ONCE(1, "verifier backtracking bug");
                                return -EFAULT;
                        }
                } else if (opcode == BPF_EXIT) {
                        bool r0_precise;

                        /* Backtracking to a nested function call, 'idx' is a part of
                         * the inner frame 'subseq_idx' is a part of the outer frame.
                         * In case of a regular function call, instructions giving
                         * precision to registers R1-R5 should have been found already.
                         * In case of a callback, it is ok to have R1-R5 marked for
                         * backtracking, as these registers are set by the function
                         * invoking callback.
                         */
                        if (subseq_idx >= 0 && calls_callback(env, subseq_idx))
                                for (i = BPF_REG_1; i <= BPF_REG_5; i++)
                                        bt_clear_reg(bt, i);
                        if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
                                verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
                                WARN_ONCE(1, "verifier backtracking bug");
                                return -EFAULT;
                        }

                        /* BPF_EXIT in subprog or callback always returns
                         * right after the call instruction, so by checking
                         * whether the instruction at subseq_idx-1 is subprog
                         * call or not we can distinguish actual exit from
                         * *subprog* from exit from *callback*. In the former
                         * case, we need to propagate r0 precision, if
                         * necessary. In the former we never do that.
                         */
                        r0_precise = subseq_idx - 1 >= 0 &&
                                     bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
                                     bt_is_reg_set(bt, BPF_REG_0);

                        bt_clear_reg(bt, BPF_REG_0);
                        if (bt_subprog_enter(bt))
                                return -EFAULT;

                        if (r0_precise)
                                bt_set_reg(bt, BPF_REG_0);
                        /* r6-r9 and stack slots will stay set in caller frame
                         * bitmasks until we return back from callee(s)
                         */
                        return 0;
                } else if (BPF_SRC(insn->code) == BPF_X) {
                        if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg))
                                return 0;
                        /* dreg <cond> sreg
                         * Both dreg and sreg need precision before
                         * this insn. If only sreg was marked precise
                         * before it would be equally necessary to
                         * propagate it to dreg.
                         */
                        bt_set_reg(bt, dreg);
                        bt_set_reg(bt, sreg);
                } else if (BPF_SRC(insn->code) == BPF_K) {
                         /* dreg <cond> K
                          * Only dreg still needs precision before
                          * this insn, so for the K-based conditional
                          * there is nothing new to be marked.
                          */
                }
        } else if (class == BPF_LD) {
                if (!bt_is_reg_set(bt, dreg))
                        return 0;
                bt_clear_reg(bt, dreg);
                /* It's ld_imm64 or ld_abs or ld_ind.
                 * For ld_imm64 no further tracking of precision
                 * into parent is necessary
                 */
                if (mode == BPF_IND || mode == BPF_ABS)
                        /* to be analyzed */
                        return -ENOTSUPP;
        }
        /* Propagate precision marks to linked registers, to account for
         * registers marked as precise in this function.
         */
        bt_sync_linked_regs(bt, hist);
        return 0;
}

/* the scalar precision tracking algorithm:
 * . at the start all registers have precise=false.
 * . scalar ranges are tracked as normal through alu and jmp insns.
 * . once precise value of the scalar register is used in:
 *   .  ptr + scalar alu
 *   . if (scalar cond K|scalar)
 *   .  helper_call(.., scalar, ...) where ARG_CONST is expected
 *   backtrack through the verifier states and mark all registers and
 *   stack slots with spilled constants that these scalar regisers
 *   should be precise.
 * . during state pruning two registers (or spilled stack slots)
 *   are equivalent if both are not precise.
 *
 * Note the verifier cannot simply walk register parentage chain,
 * since many different registers and stack slots could have been
 * used to compute single precise scalar.
 *
 * The approach of starting with precise=true for all registers and then
 * backtrack to mark a register as not precise when the verifier detects
 * that program doesn't care about specific value (e.g., when helper
 * takes register as ARG_ANYTHING parameter) is not safe.
 *
 * It's ok to walk single parentage chain of the verifier states.
 * It's possible that this backtracking will go all the way till 1st insn.
 * All other branches will be explored for needing precision later.
 *
 * The backtracking needs to deal with cases like:
 *   R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
 * r9 -= r8
 * r5 = r9
 * if r5 > 0x79f goto pc+7
 *    R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
 * r5 += 1
 * ...
 * call bpf_perf_event_output#25
 *   where .arg5_type = ARG_CONST_SIZE_OR_ZERO
 *
 * and this case:
 * r6 = 1
 * call foo // uses callee's r6 inside to compute r0
 * r0 += r6
 * if r0 == 0 goto
 *
 * to track above reg_mask/stack_mask needs to be independent for each frame.
 *
 * Also if parent's curframe > frame where backtracking started,
 * the verifier need to mark registers in both frames, otherwise callees
 * may incorrectly prune callers. This is similar to
 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
 *
 * For now backtracking falls back into conservative marking.
 */
static void mark_all_scalars_precise(struct bpf_verifier_env *env,
                                     struct bpf_verifier_state *st)
{
        struct bpf_func_state *func;
        struct bpf_reg_state *reg;
        int i, j;

        if (env->log.level & BPF_LOG_LEVEL2) {
                verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",
                        st->curframe);
        }

        /* big hammer: mark all scalars precise in this path.
         * pop_stack may still get !precise scalars.
         * We also skip current state and go straight to first parent state,
         * because precision markings in current non-checkpointed state are
         * not needed. See why in the comment in __mark_chain_precision below.
         */
        for (st = st->parent; st; st = st->parent) {
                for (i = 0; i <= st->curframe; i++) {
                        func = st->frame[i];
                        for (j = 0; j < BPF_REG_FP; j++) {
                                reg = &func->regs[j];
                                if (reg->type != SCALAR_VALUE || reg->precise)
                                        continue;
                                reg->precise = true;
                                if (env->log.level & BPF_LOG_LEVEL2) {
                                        verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",
                                                i, j);
                                }
                        }
                        for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
                                if (!is_spilled_reg(&func->stack[j]))
                                        continue;
                                reg = &func->stack[j].spilled_ptr;
                                if (reg->type != SCALAR_VALUE || reg->precise)
                                        continue;
                                reg->precise = true;
                                if (env->log.level & BPF_LOG_LEVEL2) {
                                        verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",
                                                i, -(j + 1) * 8);
                                }
                        }
                }
        }
}

static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
{
        struct bpf_func_state *func;
        struct bpf_reg_state *reg;
        int i, j;

        for (i = 0; i <= st->curframe; i++) {
                func = st->frame[i];
                for (j = 0; j < BPF_REG_FP; j++) {
                        reg = &func->regs[j];
                        if (reg->type != SCALAR_VALUE)
                                continue;
                        reg->precise = false;
                }
                for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
                        if (!is_spilled_reg(&func->stack[j]))
                                continue;
                        reg = &func->stack[j].spilled_ptr;
                        if (reg->type != SCALAR_VALUE)
                                continue;
                        reg->precise = false;
                }
        }
}

/*
 * __mark_chain_precision() backtracks BPF program instruction sequence and
 * chain of verifier states making sure that register *regno* (if regno >= 0)
 * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked
 * SCALARS, as well as any other registers and slots that contribute to
 * a tracked state of given registers/stack slots, depending on specific BPF
 * assembly instructions (see backtrack_insns() for exact instruction handling
 * logic). This backtracking relies on recorded insn_hist and is able to
 * traverse entire chain of parent states. This process ends only when all the
 * necessary registers/slots and their transitive dependencies are marked as
 * precise.
 *
 * One important and subtle aspect is that precise marks *do not matter* in
 * the currently verified state (current state). It is important to understand
 * why this is the case.
 *
 * First, note that current state is the state that is not yet "checkpointed",
 * i.e., it is not yet put into env->explored_states, and it has no children
 * states as well. It's ephemeral, and can end up either a) being discarded if
 * compatible explored state is found at some point or BPF_EXIT instruction is
 * reached or b) checkpointed and put into env->explored_states, branching out
 * into one or more children states.
 *
 * In the former case, precise markings in current state are completely
 * ignored by state comparison code (see regsafe() for details). Only
 * checkpointed ("old") state precise markings are important, and if old
 * state's register/slot is precise, regsafe() assumes current state's
 * register/slot as precise and checks value ranges exactly and precisely. If
 * states turn out to be compatible, current state's necessary precise
 * markings and any required parent states' precise markings are enforced
 * after the fact with propagate_precision() logic, after the fact. But it's
 * important to realize that in this case, even after marking current state
 * registers/slots as precise, we immediately discard current state. So what
 * actually matters is any of the precise markings propagated into current
 * state's parent states, which are always checkpointed (due to b) case above).
 * As such, for scenario a) it doesn't matter if current state has precise
 * markings set or not.
 *
 * Now, for the scenario b), checkpointing and forking into child(ren)
 * state(s). Note that before current state gets to checkpointing step, any
 * processed instruction always assumes precise SCALAR register/slot
 * knowledge: if precise value or range is useful to prune jump branch, BPF
 * verifier takes this opportunity enthusiastically. Similarly, when
 * register's value is used to calculate offset or memory address, exact
 * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to
 * what we mentioned above about state comparison ignoring precise markings
 * during state comparison, BPF verifier ignores and also assumes precise
 * markings *at will* during instruction verification process. But as verifier
 * assumes precision, it also propagates any precision dependencies across
 * parent states, which are not yet finalized, so can be further restricted
 * based on new knowledge gained from restrictions enforced by their children
 * states. This is so that once those parent states are finalized, i.e., when
 * they have no more active children state, state comparison logic in
 * is_state_visited() would enforce strict and precise SCALAR ranges, if
 * required for correctness.
 *
 * To build a bit more intuition, note also that once a state is checkpointed,
 * the path we took to get to that state is not important. This is crucial
 * property for state pruning. When state is checkpointed and finalized at
 * some instruction index, it can be correctly and safely used to "short
 * circuit" any *compatible* state that reaches exactly the same instruction
 * index. I.e., if we jumped to that instruction from a completely different
 * code path than original finalized state was derived from, it doesn't
 * matter, current state can be discarded because from that instruction
 * forward having a compatible state will ensure we will safely reach the
 * exit. States describe preconditions for further exploration, but completely
 * forget the history of how we got here.
 *
 * This also means that even if we needed precise SCALAR range to get to
 * finalized state, but from that point forward *that same* SCALAR register is
 * never used in a precise context (i.e., it's precise value is not needed for
 * correctness), it's correct and safe to mark such register as "imprecise"
 * (i.e., precise marking set to false). This is what we rely on when we do
 * not set precise marking in current state. If no child state requires
 * precision for any given SCALAR register, it's safe to dictate that it can
 * be imprecise. If any child state does require this register to be precise,
 * we'll mark it precise later retroactively during precise markings
 * propagation from child state to parent states.
 *
 * Skipping precise marking setting in current state is a mild version of
 * relying on the above observation. But we can utilize this property even
 * more aggressively by proactively forgetting any precise marking in the
 * current state (which we inherited from the parent state), right before we
 * checkpoint it and branch off into new child state. This is done by
 * mark_all_scalars_imprecise() to hopefully get more permissive and generic
 * finalized states which help in short circuiting more future states.
 */
static int __mark_chain_precision(struct bpf_verifier_env *env, int regno)
{
        struct backtrack_state *bt = &env->bt;
        struct bpf_verifier_state *st = env->cur_state;
        int first_idx = st->first_insn_idx;
        int last_idx = env->insn_idx;
        int subseq_idx = -1;
        struct bpf_func_state *func;
        struct bpf_reg_state *reg;
        bool skip_first = true;
        int i, fr, err;

        if (!env->bpf_capable)
                return 0;

        /* set frame number from which we are starting to backtrack */
        bt_init(bt, env->cur_state->curframe);

        /* Do sanity checks against current state of register and/or stack
         * slot, but don't set precise flag in current state, as precision
         * tracking in the current state is unnecessary.
         */
        func = st->frame[bt->frame];
        if (regno >= 0) {
                reg = &func->regs[regno];
                if (reg->type != SCALAR_VALUE) {
                        WARN_ONCE(1, "backtracing misuse");
                        return -EFAULT;
                }
                bt_set_reg(bt, regno);
        }

        if (bt_empty(bt))
                return 0;

        for (;;) {
                DECLARE_BITMAP(mask, 64);
                u32 hist_start = st->insn_hist_start;
                u32 hist_end = st->insn_hist_end;
                struct bpf_insn_hist_entry *hist;

                if (env->log.level & BPF_LOG_LEVEL2) {
                        verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
                                bt->frame, last_idx, first_idx, subseq_idx);
                }

                if (last_idx < 0) {
                        /* we are at the entry into subprog, which
                         * is expected for global funcs, but only if
                         * requested precise registers are R1-R5
                         * (which are global func's input arguments)
                         */
                        if (st->curframe == 0 &&
                            st->frame[0]->subprogno > 0 &&
                            st->frame[0]->callsite == BPF_MAIN_FUNC &&
                            bt_stack_mask(bt) == 0 &&
                            (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) {
                                bitmap_from_u64(mask, bt_reg_mask(bt));
                                for_each_set_bit(i, mask, 32) {
                                        reg = &st->frame[0]->regs[i];
                                        bt_clear_reg(bt, i);
                                        if (reg->type == SCALAR_VALUE)
                                                reg->precise = true;
                                }
                                return 0;
                        }

                        verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n",
                                st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
                        WARN_ONCE(1, "verifier backtracking bug");
                        return -EFAULT;
                }

                for (i = last_idx;;) {
                        if (skip_first) {
                                err = 0;
                                skip_first = false;
                        } else {
                                hist = get_insn_hist_entry(env, hist_start, hist_end, i);
                                err = backtrack_insn(env, i, subseq_idx, hist, bt);
                        }
                        if (err == -ENOTSUPP) {
                                mark_all_scalars_precise(env, env->cur_state);
                                bt_reset(bt);
                                return 0;
                        } else if (err) {
                                return err;
                        }
                        if (bt_empty(bt))
                                /* Found assignment(s) into tracked register in this state.
                                 * Since this state is already marked, just return.
                                 * Nothing to be tracked further in the parent state.
                                 */
                                return 0;
                        subseq_idx = i;
                        i = get_prev_insn_idx(env, st, i, hist_start, &hist_end);
                        if (i == -ENOENT)
                                break;
                        if (i >= env->prog->len) {
                                /* This can happen if backtracking reached insn 0
                                 * and there are still reg_mask or stack_mask
                                 * to backtrack.
                                 * It means the backtracking missed the spot where
                                 * particular register was initialized with a constant.
                                 */
                                verbose(env, "BUG backtracking idx %d\n", i);
                                WARN_ONCE(1, "verifier backtracking bug");
                                return -EFAULT;
                        }
                }
                st = st->parent;
                if (!st)
                        break;

                for (fr = bt->frame; fr >= 0; fr--) {
                        func = st->frame[fr];
                        bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
                        for_each_set_bit(i, mask, 32) {
                                reg = &func->regs[i];
                                if (reg->type != SCALAR_VALUE) {
                                        bt_clear_frame_reg(bt, fr, i);
                                        continue;
                                }
                                if (reg->precise)
                                        bt_clear_frame_reg(bt, fr, i);
                                else
                                        reg->precise = true;
                        }

                        bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
                        for_each_set_bit(i, mask, 64) {
                                if (i >= func->allocated_stack / BPF_REG_SIZE) {
                                        verbose(env, "BUG backtracking (stack slot %d, total slots %d)\n",
                                                i, func->allocated_stack / BPF_REG_SIZE);
                                        WARN_ONCE(1, "verifier backtracking bug (stack slot out of bounds)");
                                        return -EFAULT;
                                }

                                if (!is_spilled_scalar_reg(&func->stack[i])) {
                                        bt_clear_frame_slot(bt, fr, i);
                                        continue;
                                }
                                reg = &func->stack[i].spilled_ptr;
                                if (reg->precise)
                                        bt_clear_frame_slot(bt, fr, i);
                                else
                                        reg->precise = true;
                        }
                        if (env->log.level & BPF_LOG_LEVEL2) {
                                fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
                                             bt_frame_reg_mask(bt, fr));
                                verbose(env, "mark_precise: frame%d: parent state regs=%s ",
                                        fr, env->tmp_str_buf);
                                fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
                                               bt_frame_stack_mask(bt, fr));
                                verbose(env, "stack=%s: ", env->tmp_str_buf);
                                print_verifier_state(env, st, fr, true);
                        }
                }

                if (bt_empty(bt))
                        return 0;

                subseq_idx = first_idx;
                last_idx = st->last_insn_idx;
                first_idx = st->first_insn_idx;
        }

        /* if we still have requested precise regs or slots, we missed
         * something (e.g., stack access through non-r10 register), so
         * fallback to marking all precise
         */
        if (!bt_empty(bt)) {
                mark_all_scalars_precise(env, env->cur_state);
                bt_reset(bt);
        }

        return 0;
}

int mark_chain_precision(struct bpf_verifier_env *env, int regno)
{
        return __mark_chain_precision(env, regno);
}

/* mark_chain_precision_batch() assumes that env->bt is set in the caller to
 * desired reg and stack masks across all relevant frames
 */
static int mark_chain_precision_batch(struct bpf_verifier_env *env)
{
        return __mark_chain_precision(env, -1);
}

static bool is_spillable_regtype(enum bpf_reg_type type)
{
        switch (base_type(type)) {
        case PTR_TO_MAP_VALUE:
        case PTR_TO_STACK:
        case PTR_TO_CTX:
        case PTR_TO_PACKET:
        case PTR_TO_PACKET_META:
        case PTR_TO_PACKET_END:
        case PTR_TO_FLOW_KEYS:
        case CONST_PTR_TO_MAP:
        case PTR_TO_SOCKET:
        case PTR_TO_SOCK_COMMON:
        case PTR_TO_TCP_SOCK:
        case PTR_TO_XDP_SOCK:
        case PTR_TO_BTF_ID:
        case PTR_TO_BUF:
        case PTR_TO_MEM:
        case PTR_TO_FUNC:
        case PTR_TO_MAP_KEY:
        case PTR_TO_ARENA:
                return true;
        default:
                return false;
        }
}

/* Does this register contain a constant zero? */
static bool register_is_null(struct bpf_reg_state *reg)
{
        return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
}

/* check if register is a constant scalar value */
static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32)
{
        return reg->type == SCALAR_VALUE &&
               tnum_is_const(subreg32 ? tnum_subreg(reg->var_off) : reg->var_off);
}

/* assuming is_reg_const() is true, return constant value of a register */
static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32)
{
        return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value;
}

static bool __is_pointer_value(bool allow_ptr_leaks,
                               const struct bpf_reg_state *reg)
{
        if (allow_ptr_leaks)
                return false;

        return reg->type != SCALAR_VALUE;
}

static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,
                                        struct bpf_reg_state *src_reg)
{
        if (src_reg->type != SCALAR_VALUE)
                return;

        if (src_reg->id & BPF_ADD_CONST) {
                /*
                 * The verifier is processing rX = rY insn and
                 * rY->id has special linked register already.
                 * Cleared it, since multiple rX += const are not supported.
                 */
                src_reg->id = 0;
                src_reg->off = 0;
        }

        if (!src_reg->id && !tnum_is_const(src_reg->var_off))
                /* Ensure that src_reg has a valid ID that will be copied to
                 * dst_reg and then will be used by sync_linked_regs() to
                 * propagate min/max range.
                 */
                src_reg->id = ++env->id_gen;
}

/* Copy src state preserving dst->parent and dst->live fields */
static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src)
{
        struct bpf_reg_state *parent = dst->parent;
        enum bpf_reg_liveness live = dst->live;

        *dst = *src;
        dst->parent = parent;
        dst->live = live;
}

static void save_register_state(struct bpf_verifier_env *env,
                                struct bpf_func_state *state,
                                int spi, struct bpf_reg_state *reg,
                                int size)
{
        int i;

        copy_register_state(&state->stack[spi].spilled_ptr, reg);
        if (size == BPF_REG_SIZE)
                state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;

        for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
                state->stack[spi].slot_type[i - 1] = STACK_SPILL;

        /* size < 8 bytes spill */
        for (; i; i--)
                mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);
}

static bool is_bpf_st_mem(struct bpf_insn *insn)
{
        return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM;
}

static int get_reg_width(struct bpf_reg_state *reg)
{
        return fls64(reg->umax_value);
}

/* See comment for mark_fastcall_pattern_for_call() */
static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
                                          struct bpf_func_state *state, int insn_idx, int off)
{
        struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
        struct bpf_insn_aux_data *aux = env->insn_aux_data;
        int i;

        if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern)
                return;
        /* access to the region [max_stack_depth .. fastcall_stack_off)
         * from something that is not a part of the fastcall pattern,
         * disable fastcall rewrites for current subprogram by setting
         * fastcall_stack_off to a value smaller than any possible offset.
         */
        subprog->fastcall_stack_off = S16_MIN;
        /* reset fastcall aux flags within subprogram,
         * happens at most once per subprogram
         */
        for (i = subprog->start; i < (subprog + 1)->start; ++i) {
                aux[i].fastcall_spills_num = 0;
                aux[i].fastcall_pattern = 0;
        }
}

/* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
 * stack boundary and alignment are checked in check_mem_access()
 */
static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
                                       /* stack frame we're writing to */
                                       struct bpf_func_state *state,
                                       int off, int size, int value_regno,
                                       int insn_idx)
{
        struct bpf_func_state *cur; /* state of the current function */
        int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
        struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
        struct bpf_reg_state *reg = NULL;
        int insn_flags = insn_stack_access_flags(state->frameno, spi);

        /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
         * so it's aligned access and [off, off + size) are within stack limits
         */
        if (!env->allow_ptr_leaks &&
            is_spilled_reg(&state->stack[spi]) &&
            !is_spilled_scalar_reg(&state->stack[spi]) &&
            size != BPF_REG_SIZE) {
                verbose(env, "attempt to corrupt spilled pointer on stack\n");
                return -EACCES;
        }

        cur = env->cur_state->frame[env->cur_state->curframe];
        if (value_regno >= 0)
                reg = &cur->regs[value_regno];
        if (!env->bypass_spec_v4) {
                bool sanitize = reg && is_spillable_regtype(reg->type);

                for (i = 0; i < size; i++) {
                        u8 type = state->stack[spi].slot_type[i];

                        if (type != STACK_MISC && type != STACK_ZERO) {
                                sanitize = true;
                                break;
                        }
                }

                if (sanitize)
                        env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
        }

        err = destroy_if_dynptr_stack_slot(env, state, spi);
        if (err)
                return err;

        check_fastcall_stack_contract(env, state, insn_idx, off);
        mark_stack_slot_scratched(env, spi);
        if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {
                bool reg_value_fits;

                reg_value_fits = get_reg_width(reg) <= BITS_PER_BYTE * size;
                /* Make sure that reg had an ID to build a relation on spill. */
                if (reg_value_fits)
                        assign_scalar_id_before_mov(env, reg);
                save_register_state(env, state, spi, reg, size);
                /* Break the relation on a narrowing spill. */
                if (!reg_value_fits)
                        state->stack[spi].spilled_ptr.id = 0;
        } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) &&
                   env->bpf_capable) {
                struct bpf_reg_state *tmp_reg = &env->fake_reg[0];

                memset(tmp_reg, 0, sizeof(*tmp_reg));
                __mark_reg_known(tmp_reg, insn->imm);
                tmp_reg->type = SCALAR_VALUE;
                save_register_state(env, state, spi, tmp_reg, size);
        } else if (reg && is_spillable_regtype(reg->type)) {
                /* register containing pointer is being spilled into stack */
                if (size != BPF_REG_SIZE) {
                        verbose_linfo(env, insn_idx, "; ");
                        verbose(env, "invalid size of register spill\n");
                        return -EACCES;
                }
                if (state != cur && reg->type == PTR_TO_STACK) {
                        verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
                        return -EINVAL;
                }
                save_register_state(env, state, spi, reg, size);
        } else {
                u8 type = STACK_MISC;

                /* regular write of data into stack destroys any spilled ptr */
                state->stack[spi].spilled_ptr.type = NOT_INIT;
                /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */
                if (is_stack_slot_special(&state->stack[spi]))
                        for (i = 0; i < BPF_REG_SIZE; i++)
                                scrub_spilled_slot(&state->stack[spi].slot_type[i]);

                /* only mark the slot as written if all 8 bytes were written
                 * otherwise read propagation may incorrectly stop too soon
                 * when stack slots are partially written.
                 * This heuristic means that read propagation will be
                 * conservative, since it will add reg_live_read marks
                 * to stack slots all the way to first state when programs
                 * writes+reads less than 8 bytes
                 */
                if (size == BPF_REG_SIZE)
                        state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;

                /* when we zero initialize stack slots mark them as such */
                if ((reg && register_is_null(reg)) ||
                    (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) {
                        /* STACK_ZERO case happened because register spill
                         * wasn't properly aligned at the stack slot boundary,
                         * so it's not a register spill anymore; force
                         * originating register to be precise to make
                         * STACK_ZERO correct for subsequent states
                         */
                        err = mark_chain_precision(env, value_regno);
                        if (err)
                                return err;
                        type = STACK_ZERO;
                }

                /* Mark slots affected by this stack write. */
                for (i = 0; i < size; i++)
                        state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type;
                insn_flags = 0; /* not a register spill */
        }

        if (insn_flags)
                return push_insn_history(env, env->cur_state, insn_flags, 0);
        return 0;
}

/* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
 * known to contain a variable offset.
 * This function checks whether the write is permitted and conservatively
 * tracks the effects of the write, considering that each stack slot in the
 * dynamic range is potentially written to.
 *
 * 'off' includes 'regno->off'.
 * 'value_regno' can be -1, meaning that an unknown value is being written to
 * the stack.
 *
 * Spilled pointers in range are not marked as written because we don't know
 * what's going to be actually written. This means that read propagation for
 * future reads cannot be terminated by this write.
 *
 * For privileged programs, uninitialized stack slots are considered
 * initialized by this write (even though we don't know exactly what offsets
 * are going to be written to). The idea is that we don't want the verifier to
 * reject future reads that access slots written to through variable offsets.
 */
static int check_stack_write_var_off(struct bpf_verifier_env *env,
                                     /* func where register points to */
                                     struct bpf_func_state *state,
                                     int ptr_regno, int off, int size,
                                     int value_regno, int insn_idx)
{
        struct bpf_func_state *cur; /* state of the current function */
        int min_off, max_off;
        int i, err;
        struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
        struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
        bool writing_zero = false;
        /* set if the fact that we're writing a zero is used to let any
         * stack slots remain STACK_ZERO
         */
        bool zero_used = false;

        cur = env->cur_state->frame[env->cur_state->curframe];
        ptr_reg = &cur->regs[ptr_regno];
        min_off = ptr_reg->smin_value + off;
        max_off = ptr_reg->smax_value + off + size;
        if (value_regno >= 0)
                value_reg = &cur->regs[value_regno];
        if ((value_reg && register_is_null(value_reg)) ||
            (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0))
                writing_zero = true;

        for (i = min_off; i < max_off; i++) {
                int spi;

                spi = __get_spi(i);
                err = destroy_if_dynptr_stack_slot(env, state, spi);
                if (err)
                        return err;
        }

        check_fastcall_stack_contract(env, state, insn_idx, min_off);
        /* Variable offset writes destroy any spilled pointers in range. */
        for (i = min_off; i < max_off; i++) {
                u8 new_type, *stype;
                int slot, spi;

                slot = -i - 1;
                spi = slot / BPF_REG_SIZE;
                stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
                mark_stack_slot_scratched(env, spi);

                if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {
                        /* Reject the write if range we may write to has not
                         * been initialized beforehand. If we didn't reject
                         * here, the ptr status would be erased below (even
                         * though not all slots are actually overwritten),
                         * possibly opening the door to leaks.
                         *
                         * We do however catch STACK_INVALID case below, and
                         * only allow reading possibly uninitialized memory
                         * later for CAP_PERFMON, as the write may not happen to
                         * that slot.
                         */
                        verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
                                insn_idx, i);
                        return -EINVAL;
                }

                /* If writing_zero and the spi slot contains a spill of value 0,
                 * maintain the spill type.
                 */
                if (writing_zero && *stype == STACK_SPILL &&
                    is_spilled_scalar_reg(&state->stack[spi])) {
                        struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr;

                        if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) {
                                zero_used = true;
                                continue;
                        }
                }

                /* Erase all other spilled pointers. */
                state->stack[spi].spilled_ptr.type = NOT_INIT;

                /* Update the slot type. */
                new_type = STACK_MISC;
                if (writing_zero && *stype == STACK_ZERO) {
                        new_type = STACK_ZERO;
                        zero_used = true;
                }
                /* If the slot is STACK_INVALID, we check whether it's OK to
                 * pretend that it will be initialized by this write. The slot
                 * might not actually be written to, and so if we mark it as
                 * initialized future reads might leak uninitialized memory.
                 * For privileged programs, we will accept such reads to slots
                 * that may or may not be written because, if we're reject
                 * them, the error would be too confusing.
                 */
                if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
                        verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
                                        insn_idx, i);
                        return -EINVAL;
                }
                *stype = new_type;
        }
        if (zero_used) {
                /* backtracking doesn't work for STACK_ZERO yet. */
                err = mark_chain_precision(env, value_regno);
                if (err)
                        return err;
        }
        return 0;
}

/* When register 'dst_regno' is assigned some values from stack[min_off,
 * max_off), we set the register's type according to the types of the
 * respective stack slots. If all the stack values are known to be zeros, then
 * so is the destination reg. Otherwise, the register is considered to be
 * SCALAR. This function does not deal with register filling; the caller must
 * ensure that all spilled registers in the stack range have been marked as
 * read.
 */
static void mark_reg_stack_read(struct bpf_verifier_env *env,
                                /* func where src register points to */
                                struct bpf_func_state *ptr_state,
                                int min_off, int max_off, int dst_regno)
{
        struct bpf_verifier_state *vstate = env->cur_state;
        struct bpf_func_state *state = vstate->frame[vstate->curframe];
        int i, slot, spi;
        u8 *stype;
        int zeros = 0;

        for (i = min_off; i < max_off; i++) {
                slot = -i - 1;
                spi = slot / BPF_REG_SIZE;
                mark_stack_slot_scratched(env, spi);
                stype = ptr_state->stack[spi].slot_type;
                if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
                        break;
                zeros++;
        }
        if (zeros == max_off - min_off) {
                /* Any access_size read into register is zero extended,
                 * so the whole register == const_zero.
                 */
                __mark_reg_const_zero(env, &state->regs[dst_regno]);
        } else {
                /* have read misc data from the stack */
                mark_reg_unknown(env, state->regs, dst_regno);
        }
        state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
}

/* Read the stack at 'off' and put the results into the register indicated by
 * 'dst_regno'. It handles reg filling if the addressed stack slot is a
 * spilled reg.
 *
 * 'dst_regno' can be -1, meaning that the read value is not going to a
 * register.
 *
 * The access is assumed to be within the current stack bounds.
 */
static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
                                      /* func where src register points to */
                                      struct bpf_func_state *reg_state,
                                      int off, int size, int dst_regno)
{
        struct bpf_verifier_state *vstate = env->cur_state;
        struct bpf_func_state *state = vstate->frame[vstate->curframe];
        int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
        struct bpf_reg_state *reg;
        u8 *stype, type;
        int insn_flags = insn_stack_access_flags(reg_state->frameno, spi);

        stype = reg_state->stack[spi].slot_type;
        reg = &reg_state->stack[spi].spilled_ptr;

        mark_stack_slot_scratched(env, spi);
        check_fastcall_stack_contract(env, state, env->insn_idx, off);

        if (is_spilled_reg(&reg_state->stack[spi])) {
                u8 spill_size = 1;

                for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--)
                        spill_size++;

                if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) {
                        if (reg->type != SCALAR_VALUE) {
                                verbose_linfo(env, env->insn_idx, "; ");
                                verbose(env, "invalid size of register fill\n");
                                return -EACCES;
                        }

                        mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
                        if (dst_regno < 0)
                                return 0;

                        if (size <= spill_size &&
                            bpf_stack_narrow_access_ok(off, size, spill_size)) {
                                /* The earlier check_reg_arg() has decided the
                                 * subreg_def for this insn.  Save it first.
                                 */
                                s32 subreg_def = state->regs[dst_regno].subreg_def;

                                copy_register_state(&state->regs[dst_regno], reg);
                                state->regs[dst_regno].subreg_def = subreg_def;

                                /* Break the relation on a narrowing fill.
                                 * coerce_reg_to_size will adjust the boundaries.
                                 */
                                if (get_reg_width(reg) > size * BITS_PER_BYTE)
                                        state->regs[dst_regno].id = 0;
                        } else {
                                int spill_cnt = 0, zero_cnt = 0;

                                for (i = 0; i < size; i++) {
                                        type = stype[(slot - i) % BPF_REG_SIZE];
                                        if (type == STACK_SPILL) {
                                                spill_cnt++;
                                                continue;
                                        }
                                        if (type == STACK_MISC)
                                                continue;
                                        if (type == STACK_ZERO) {
                                                zero_cnt++;
                                                continue;
                                        }
                                        if (type == STACK_INVALID && env->allow_uninit_stack)
                                                continue;
                                        verbose(env, "invalid read from stack off %d+%d size %d\n",
                                                off, i, size);
                                        return -EACCES;
                                }

                                if (spill_cnt == size &&
                                    tnum_is_const(reg->var_off) && reg->var_off.value == 0) {
                                        __mark_reg_const_zero(env, &state->regs[dst_regno]);
                                        /* this IS register fill, so keep insn_flags */
                                } else if (zero_cnt == size) {
                                        /* similarly to mark_reg_stack_read(), preserve zeroes */
                                        __mark_reg_const_zero(env, &state->regs[dst_regno]);
                                        insn_flags = 0; /* not restoring original register state */
                                } else {
                                        mark_reg_unknown(env, state->regs, dst_regno);
                                        insn_flags = 0; /* not restoring original register state */
                                }
                        }
                        state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
                } else if (dst_regno >= 0) {
                        /* restore register state from stack */
                        copy_register_state(&state->regs[dst_regno], reg);
                        /* mark reg as written since spilled pointer state likely
                         * has its liveness marks cleared by is_state_visited()
                         * which resets stack/reg liveness for state transitions
                         */
                        state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
                } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
                        /* If dst_regno==-1, the caller is asking us whether
                         * it is acceptable to use this value as a SCALAR_VALUE
                         * (e.g. for XADD).
                         * We must not allow unprivileged callers to do that
                         * with spilled pointers.
                         */
                        verbose(env, "leaking pointer from stack off %d\n",
                                off);
                        return -EACCES;
                }
                mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
        } else {
                for (i = 0; i < size; i++) {
                        type = stype[(slot - i) % BPF_REG_SIZE];
                        if (type == STACK_MISC)
                                continue;
                        if (type == STACK_ZERO)
                                continue;
                        if (type == STACK_INVALID && env->allow_uninit_stack)
                                continue;
                        verbose(env, "invalid read from stack off %d+%d size %d\n",
                                off, i, size);
                        return -EACCES;
                }
                mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
                if (dst_regno >= 0)
                        mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
                insn_flags = 0; /* we are not restoring spilled register */
        }
        if (insn_flags)
                return push_insn_history(env, env->cur_state, insn_flags, 0);
        return 0;
}

enum bpf_access_src {
        ACCESS_DIRECT = 1,  /* the access is performed by an instruction */
        ACCESS_HELPER = 2,  /* the access is performed by a helper */
};

static int check_stack_range_initialized(struct bpf_verifier_env *env,
                                         int regno, int off, int access_size,
                                         bool zero_size_allowed,
                                         enum bpf_access_type type,
                                         struct bpf_call_arg_meta *meta);

static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
{
        return cur_regs(env) + regno;
}

/* Read the stack at 'ptr_regno + off' and put the result into the register
 * 'dst_regno'.
 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
 * but not its variable offset.
 * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
 *
 * As opposed to check_stack_read_fixed_off, this function doesn't deal with
 * filling registers (i.e. reads of spilled register cannot be detected when
 * the offset is not fixed). We conservatively mark 'dst_regno' as containing
 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
 * offset; for a fixed offset check_stack_read_fixed_off should be used
 * instead.
 */
static int check_stack_read_var_off(struct bpf_verifier_env *env,
                                    int ptr_regno, int off, int size, int dst_regno)
{
        /* The state of the source register. */
        struct bpf_reg_state *reg = reg_state(env, ptr_regno);
        struct bpf_func_state *ptr_state = func(env, reg);
        int err;
        int min_off, max_off;

        /* Note that we pass a NULL meta, so raw access will not be permitted.
         */
        err = check_stack_range_initialized(env, ptr_regno, off, size,
                                            false, BPF_READ, NULL);
        if (err)
                return err;

        min_off = reg->smin_value + off;
        max_off = reg->smax_value + off;
        mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
        check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);
        return 0;
}

/* check_stack_read dispatches to check_stack_read_fixed_off or
 * check_stack_read_var_off.
 *
 * The caller must ensure that the offset falls within the allocated stack
 * bounds.
 *
 * 'dst_regno' is a register which will receive the value from the stack. It
 * can be -1, meaning that the read value is not going to a register.
 */
static int check_stack_read(struct bpf_verifier_env *env,
                            int ptr_regno, int off, int size,
                            int dst_regno)
{
        struct bpf_reg_state *reg = reg_state(env, ptr_regno);
        struct bpf_func_state *state = func(env, reg);
        int err;
        /* Some accesses are only permitted with a static offset. */
        bool var_off = !tnum_is_const(reg->var_off);

        /* The offset is required to be static when reads don't go to a
         * register, in order to not leak pointers (see
         * check_stack_read_fixed_off).
         */
        if (dst_regno < 0 && var_off) {
                char tn_buf[48];

                tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
                        tn_buf, off, size);
                return -EACCES;
        }
        /* Variable offset is prohibited for unprivileged mode for simplicity
         * since it requires corresponding support in Spectre masking for stack
         * ALU. See also retrieve_ptr_limit(). The check in
         * check_stack_access_for_ptr_arithmetic() called by
         * adjust_ptr_min_max_vals() prevents users from creating stack pointers
         * with variable offsets, therefore no check is required here. Further,
         * just checking it here would be insufficient as speculative stack
         * writes could still lead to unsafe speculative behaviour.
         */
        if (!var_off) {
                off += reg->var_off.value;
                err = check_stack_read_fixed_off(env, state, off, size,
                                                 dst_regno);
        } else {
                /* Variable offset stack reads need more conservative handling
                 * than fixed offset ones. Note that dst_regno >= 0 on this
                 * branch.
                 */
                err = check_stack_read_var_off(env, ptr_regno, off, size,
                                               dst_regno);
        }
        return err;
}


/* check_stack_write dispatches to check_stack_write_fixed_off or
 * check_stack_write_var_off.
 *
 * 'ptr_regno' is the register used as a pointer into the stack.
 * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
 * 'value_regno' is the register whose value we're writing to the stack. It can
 * be -1, meaning that we're not writing from a register.
 *
 * The caller must ensure that the offset falls within the maximum stack size.
 */
static int check_stack_write(struct bpf_verifier_env *env,
                             int ptr_regno, int off, int size,
                             int value_regno, int insn_idx)
{
        struct bpf_reg_state *reg = reg_state(env, ptr_regno);
        struct bpf_func_state *state = func(env, reg);
        int err;

        if (tnum_is_const(reg->var_off)) {
                off += reg->var_off.value;
                err = check_stack_write_fixed_off(env, state, off, size,
                                                  value_regno, insn_idx);
        } else {
                /* Variable offset stack reads need more conservative handling
                 * than fixed offset ones.
                 */
                err = check_stack_write_var_off(env, state,
                                                ptr_regno, off, size,
                                                value_regno, insn_idx);
        }
        return err;
}

static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
                                 int off, int size, enum bpf_access_type type)
{
        struct bpf_reg_state *regs = cur_regs(env);
        struct bpf_map *map = regs[regno].map_ptr;
        u32 cap = bpf_map_flags_to_cap(map);

        if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
                verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",
                        map->value_size, off, size);
                return -EACCES;
        }

        if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
                verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",
                        map->value_size, off, size);
                return -EACCES;
        }

        return 0;
}

/* check read/write into memory region (e.g., map value, ringbuf sample, etc) */
static int __check_mem_access(struct bpf_verifier_env *env, int regno,
                              int off, int size, u32 mem_size,
                              bool zero_size_allowed)
{
        bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
        struct bpf_reg_state *reg;

        if (off >= 0 && size_ok && (u64)off + size <= mem_size)
                return 0;

        reg = &cur_regs(env)[regno];
        switch (reg->type) {
        case PTR_TO_MAP_KEY:
                verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",
                        mem_size, off, size);
                break;
        case PTR_TO_MAP_VALUE:
                verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
                        mem_size, off, size);
                break;
        case PTR_TO_PACKET:
        case PTR_TO_PACKET_META:
        case PTR_TO_PACKET_END:
                verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
                        off, size, regno, reg->id, off, mem_size);
                break;
        case PTR_TO_MEM:
        default:
                verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",
                        mem_size, off, size);
        }

        return -EACCES;
}

/* check read/write into a memory region with possible variable offset */
static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
                                   int off, int size, u32 mem_size,
                                   bool zero_size_allowed)
{
        struct bpf_verifier_state *vstate = env->cur_state;
        struct bpf_func_state *state = vstate->frame[vstate->curframe];
        struct bpf_reg_state *reg = &state->regs[regno];
        int err;

        /* We may have adjusted the register pointing to memory region, so we
         * need to try adding each of min_value and max_value to off
         * to make sure our theoretical access will be safe.
         *
         * The minimum value is only important with signed
         * comparisons where we can't assume the floor of a
         * value is 0.  If we are using signed variables for our
         * index'es we need to make sure that whatever we use
         * will have a set floor within our range.
         */
        if (reg->smin_value < 0 &&
            (reg->smin_value == S64_MIN ||
             (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
              reg->smin_value + off < 0)) {
                verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
                        regno);
                return -EACCES;
        }
        err = __check_mem_access(env, regno, reg->smin_value + off, size,
                                 mem_size, zero_size_allowed);
        if (err) {
                verbose(env, "R%d min value is outside of the allowed memory range\n",
                        regno);
                return err;
        }

        /* If we haven't set a max value then we need to bail since we can't be
         * sure we won't do bad things.
         * If reg->umax_value + off could overflow, treat that as unbounded too.
         */
        if (reg->umax_value >= BPF_MAX_VAR_OFF) {
                verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",
                        regno);
                return -EACCES;
        }
        err = __check_mem_access(env, regno, reg->umax_value + off, size,
                                 mem_size, zero_size_allowed);
        if (err) {
                verbose(env, "R%d max value is outside of the allowed memory range\n",
                        regno);
                return err;
        }

        return 0;
}

static int __check_ptr_off_reg(struct bpf_verifier_env *env,
                               const struct bpf_reg_state *reg, int regno,
                               bool fixed_off_ok)
{
        /* Access to this pointer-typed register or passing it to a helper
         * is only allowed in its original, unmodified form.
         */

        if (reg->off < 0) {
                verbose(env, "negative offset %s ptr R%d off=%d disallowed\n",
                        reg_type_str(env, reg->type), regno, reg->off);
                return -EACCES;
        }

        if (!fixed_off_ok && reg->off) {
                verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n",
                        reg_type_str(env, reg->type), regno, reg->off);
                return -EACCES;
        }

        if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
                char tn_buf[48];

                tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                verbose(env, "variable %s access var_off=%s disallowed\n",
                        reg_type_str(env, reg->type), tn_buf);
                return -EACCES;
        }

        return 0;
}

static int check_ptr_off_reg(struct bpf_verifier_env *env,
                             const struct bpf_reg_state *reg, int regno)
{
        return __check_ptr_off_reg(env, reg, regno, false);
}

static int map_kptr_match_type(struct bpf_verifier_env *env,
                               struct btf_field *kptr_field,
                               struct bpf_reg_state *reg, u32 regno)
{
        const char *targ_name = btf_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id);
        int perm_flags;
        const char *reg_name = "";

        if (btf_is_kernel(reg->btf)) {
                perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU;

                /* Only unreferenced case accepts untrusted pointers */
                if (kptr_field->type == BPF_KPTR_UNREF)
                        perm_flags |= PTR_UNTRUSTED;
        } else {
                perm_flags = PTR_MAYBE_NULL | MEM_ALLOC;
                if (kptr_field->type == BPF_KPTR_PERCPU)
                        perm_flags |= MEM_PERCPU;
        }

        if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags))
                goto bad_type;

        /* We need to verify reg->type and reg->btf, before accessing reg->btf */
        reg_name = btf_type_name(reg->btf, reg->btf_id);

        /* For ref_ptr case, release function check should ensure we get one
         * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the
         * normal store of unreferenced kptr, we must ensure var_off is zero.
         * Since ref_ptr cannot be accessed directly by BPF insns, checks for
         * reg->off and reg->ref_obj_id are not needed here.
         */
        if (__check_ptr_off_reg(env, reg, regno, true))
                return -EACCES;

        /* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and
         * we also need to take into account the reg->off.
         *
         * We want to support cases like:
         *
         * struct foo {
         *         struct bar br;
         *         struct baz bz;
         * };
         *
         * struct foo *v;
         * v = func();              // PTR_TO_BTF_ID
         * val->foo = v;      // reg->off is zero, btf and btf_id match type
         * val->bar = &v->br; // reg->off is still zero, but we need to retry with
         *                    // first member type of struct after comparison fails
         * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked
         *                    // to match type
         *
         * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off
         * is zero. We must also ensure that btf_struct_ids_match does not walk
         * the struct to match type against first member of struct, i.e. reject
         * second case from above. Hence, when type is BPF_KPTR_REF, we set
         * strict mode to true for type match.
         */
        if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
                                  kptr_field->kptr.btf, kptr_field->kptr.btf_id,
                                  kptr_field->type != BPF_KPTR_UNREF))
                goto bad_type;
        return 0;
bad_type:
        verbose(env, "invalid kptr access, R%d type=%s%s ", regno,
                reg_type_str(env, reg->type), reg_name);
        verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);
        if (kptr_field->type == BPF_KPTR_UNREF)
                verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),
                        targ_name);
        else
                verbose(env, "\n");
        return -EINVAL;
}

static bool in_sleepable(struct bpf_verifier_env *env)
{
        return env->prog->sleepable ||
               (env->cur_state && env->cur_state->in_sleepable);
}

/* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock()
 * can dereference RCU protected pointers and result is PTR_TRUSTED.
 */
static bool in_rcu_cs(struct bpf_verifier_env *env)
{
        return env->cur_state->active_rcu_lock ||
               env->cur_state->active_locks ||
               !in_sleepable(env);
}

/* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */
BTF_SET_START(rcu_protected_types)
#ifdef CONFIG_NET
BTF_ID(struct, prog_test_ref_kfunc)
#endif
#ifdef CONFIG_CGROUPS
BTF_ID(struct, cgroup)
#endif
#ifdef CONFIG_BPF_JIT
BTF_ID(struct, bpf_cpumask)
#endif
BTF_ID(struct, task_struct)
#ifdef CONFIG_CRYPTO
BTF_ID(struct, bpf_crypto_ctx)
#endif
BTF_SET_END(rcu_protected_types)

static bool rcu_protected_object(const struct btf *btf, u32 btf_id)
{
        if (!btf_is_kernel(btf))
                return true;
        return btf_id_set_contains(&rcu_protected_types, btf_id);
}

static struct btf_record *kptr_pointee_btf_record(struct btf_field *kptr_field)
{
        struct btf_struct_meta *meta;

        if (btf_is_kernel(kptr_field->kptr.btf))
                return NULL;

        meta = btf_find_struct_meta(kptr_field->kptr.btf,
                                    kptr_field->kptr.btf_id);

        return meta ? meta->record : NULL;
}

static bool rcu_safe_kptr(const struct btf_field *field)
{
        const struct btf_field_kptr *kptr = &field->kptr;

        return field->type == BPF_KPTR_PERCPU ||
               (field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id));
}

static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field)
{
        struct btf_record *rec;
        u32 ret;

        ret = PTR_MAYBE_NULL;
        if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) {
                ret |= MEM_RCU;
                if (kptr_field->type == BPF_KPTR_PERCPU)
                        ret |= MEM_PERCPU;
                else if (!btf_is_kernel(kptr_field->kptr.btf))
                        ret |= MEM_ALLOC;

                rec = kptr_pointee_btf_record(kptr_field);
                if (rec && btf_record_has_field(rec, BPF_GRAPH_NODE))
                        ret |= NON_OWN_REF;
        } else {
                ret |= PTR_UNTRUSTED;
        }

        return ret;
}

static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno,
                            struct btf_field *field)
{
        struct bpf_reg_state *reg;
        const struct btf_type *t;

        t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
        mark_reg_known_zero(env, cur_regs(env), regno);
        reg = reg_state(env, regno);
        reg->type = PTR_TO_MEM | PTR_MAYBE_NULL;
        reg->mem_size = t->size;
        reg->id = ++env->id_gen;

        return 0;
}

static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
                                 int value_regno, int insn_idx,
                                 struct btf_field *kptr_field)
{
        struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
        int class = BPF_CLASS(insn->code);
        struct bpf_reg_state *val_reg;

        /* Things we already checked for in check_map_access and caller:
         *  - Reject cases where variable offset may touch kptr
         *  - size of access (must be BPF_DW)
         *  - tnum_is_const(reg->var_off)
         *  - kptr_field->offset == off + reg->var_off.value
         */
        /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */
        if (BPF_MODE(insn->code) != BPF_MEM) {
                verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n");
                return -EACCES;
        }

        /* We only allow loading referenced kptr, since it will be marked as
         * untrusted, similar to unreferenced kptr.
         */
        if (class != BPF_LDX &&
            (kptr_field->type == BPF_KPTR_REF || kptr_field->type == BPF_KPTR_PERCPU)) {
                verbose(env, "store to referenced kptr disallowed\n");
                return -EACCES;
        }
        if (class != BPF_LDX && kptr_field->type == BPF_UPTR) {
                verbose(env, "store to uptr disallowed\n");
                return -EACCES;
        }

        if (class == BPF_LDX) {
                if (kptr_field->type == BPF_UPTR)
                        return mark_uptr_ld_reg(env, value_regno, kptr_field);

                /* We can simply mark the value_regno receiving the pointer
                 * value from map as PTR_TO_BTF_ID, with the correct type.
                 */
                mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf,
                                kptr_field->kptr.btf_id, btf_ld_kptr_type(env, kptr_field));
        } else if (class == BPF_STX) {
                val_reg = reg_state(env, value_regno);
                if (!register_is_null(val_reg) &&
                    map_kptr_match_type(env, kptr_field, val_reg, value_regno))
                        return -EACCES;
        } else if (class == BPF_ST) {
                if (insn->imm) {
                        verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n",
                                kptr_field->offset);
                        return -EACCES;
                }
        } else {
                verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n");
                return -EACCES;
        }
        return 0;
}

/* check read/write into a map element with possible variable offset */
static int check_map_access(struct bpf_verifier_env *env, u32 regno,
                            int off, int size, bool zero_size_allowed,
                            enum bpf_access_src src)
{
        struct bpf_verifier_state *vstate = env->cur_state;
        struct bpf_func_state *state = vstate->frame[vstate->curframe];
        struct bpf_reg_state *reg = &state->regs[regno];
        struct bpf_map *map = reg->map_ptr;
        struct btf_record *rec;
        int err, i;

        err = check_mem_region_access(env, regno, off, size, map->value_size,
                                      zero_size_allowed);
        if (err)
                return err;

        if (IS_ERR_OR_NULL(map->record))
                return 0;
        rec = map->record;
        for (i = 0; i < rec->cnt; i++) {
                struct btf_field *field = &rec->fields[i];
                u32 p = field->offset;

                /* If any part of a field  can be touched by load/store, reject
                 * this program. To check that [x1, x2) overlaps with [y1, y2),
                 * it is sufficient to check x1 < y2 && y1 < x2.
                 */
                if (reg->smin_value + off < p + field->size &&
                    p < reg->umax_value + off + size) {
                        switch (field->type) {
                        case BPF_KPTR_UNREF:
                        case BPF_KPTR_REF:
                        case BPF_KPTR_PERCPU:
                        case BPF_UPTR:
                                if (src != ACCESS_DIRECT) {
                                        verbose(env, "%s cannot be accessed indirectly by helper\n",
                                                btf_field_type_name(field->type));
                                        return -EACCES;
                                }
                                if (!tnum_is_const(reg->var_off)) {
                                        verbose(env, "%s access cannot have variable offset\n",
                                                btf_field_type_name(field->type));
                                        return -EACCES;
                                }
                                if (p != off + reg->var_off.value) {
                                        verbose(env, "%s access misaligned expected=%u off=%llu\n",
                                                btf_field_type_name(field->type),
                                                p, off + reg->var_off.value);
                                        return -EACCES;
                                }
                                if (size != bpf_size_to_bytes(BPF_DW)) {
                                        verbose(env, "%s access size must be BPF_DW\n",
                                                btf_field_type_name(field->type));
                                        return -EACCES;
                                }
                                break;
                        default:
                                verbose(env, "%s cannot be accessed directly by load/store\n",
                                        btf_field_type_name(field->type));
                                return -EACCES;
                        }
                }
        }
        return 0;
}

#define MAX_PACKET_OFF 0xffff

static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
                                       const struct bpf_call_arg_meta *meta,
                                       enum bpf_access_type t)
{
        enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

        switch (prog_type) {
        /* Program types only with direct read access go here! */
        case BPF_PROG_TYPE_LWT_IN:
        case BPF_PROG_TYPE_LWT_OUT:
        case BPF_PROG_TYPE_LWT_SEG6LOCAL:
        case BPF_PROG_TYPE_SK_REUSEPORT:
        case BPF_PROG_TYPE_FLOW_DISSECTOR:
        case BPF_PROG_TYPE_CGROUP_SKB:
                if (t == BPF_WRITE)
                        return false;
                fallthrough;

        /* Program types with direct read + write access go here! */
        case BPF_PROG_TYPE_SCHED_CLS:
        case BPF_PROG_TYPE_SCHED_ACT:
        case BPF_PROG_TYPE_XDP:
        case BPF_PROG_TYPE_LWT_XMIT:
        case BPF_PROG_TYPE_SK_SKB:
        case BPF_PROG_TYPE_SK_MSG:
                if (meta)
                        return meta->pkt_access;

                env->seen_direct_write = true;
                return true;

        case BPF_PROG_TYPE_CGROUP_SOCKOPT:
                if (t == BPF_WRITE)
                        env->seen_direct_write = true;

                return true;

        default:
                return false;
        }
}

static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
                               int size, bool zero_size_allowed)
{
        struct bpf_reg_state *regs = cur_regs(env);
        struct bpf_reg_state *reg = &regs[regno];
        int err;

        /* We may have added a variable offset to the packet pointer; but any
         * reg->range we have comes after that.  We are only checking the fixed
         * offset.
         */

        /* We don't allow negative numbers, because we aren't tracking enough
         * detail to prove they're safe.
         */
        if (reg->smin_value < 0) {
                verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
                        regno);
                return -EACCES;
        }

        err = reg->range < 0 ? -EINVAL :
              __check_mem_access(env, regno, off, size, reg->range,
                                 zero_size_allowed);
        if (err) {
                verbose(env, "R%d offset is outside of the packet\n", regno);
                return err;
        }

        /* __check_mem_access has made sure "off + size - 1" is within u16.
         * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
         * otherwise find_good_pkt_pointers would have refused to set range info
         * that __check_mem_access would have rejected this pkt access.
         * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
         */
        env->prog->aux->max_pkt_offset =
                max_t(u32, env->prog->aux->max_pkt_offset,
                      off + reg->umax_value + size - 1);

        return err;
}

/* check access to 'struct bpf_context' fields.  Supports fixed offsets only */
static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
                            enum bpf_access_type t, enum bpf_reg_type *reg_type,
                            struct btf **btf, u32 *btf_id, bool *is_retval, bool is_ldsx)
{
        struct bpf_insn_access_aux info = {
                .reg_type = *reg_type,
                .log = &env->log,
                .is_retval = false,
                .is_ldsx = is_ldsx,
        };

        if (env->ops->is_valid_access &&
            env->ops->is_valid_access(off, size, t, env->prog, &info)) {
                /* A non zero info.ctx_field_size indicates that this field is a
                 * candidate for later verifier transformation to load the whole
                 * field and then apply a mask when accessed with a narrower
                 * access than actual ctx access size. A zero info.ctx_field_size
                 * will only allow for whole field access and rejects any other
                 * type of narrower access.
                 */
                *reg_type = info.reg_type;
                *is_retval = info.is_retval;

                if (base_type(*reg_type) == PTR_TO_BTF_ID) {
                        *btf = info.btf;
                        *btf_id = info.btf_id;
                } else {
                        env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
                }
                /* remember the offset of last byte accessed in ctx */
                if (env->prog->aux->max_ctx_offset < off + size)
                        env->prog->aux->max_ctx_offset = off + size;
                return 0;
        }

        verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
        return -EACCES;
}

static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
                                  int size)
{
        if (size < 0 || off < 0 ||
            (u64)off + size > sizeof(struct bpf_flow_keys)) {
                verbose(env, "invalid access to flow keys off=%d size=%d\n",
                        off, size);
                return -EACCES;
        }
        return 0;
}

static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
                             u32 regno, int off, int size,
                             enum bpf_access_type t)
{
        struct bpf_reg_state *regs = cur_regs(env);
        struct bpf_reg_state *reg = &regs[regno];
        struct bpf_insn_access_aux info = {};
        bool valid;

        if (reg->smin_value < 0) {
                verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
                        regno);
                return -EACCES;
        }

        switch (reg->type) {
        case PTR_TO_SOCK_COMMON:
                valid = bpf_sock_common_is_valid_access(off, size, t, &info);
                break;
        case PTR_TO_SOCKET:
                valid = bpf_sock_is_valid_access(off, size, t, &info);
                break;
        case PTR_TO_TCP_SOCK:
                valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
                break;
        case PTR_TO_XDP_SOCK:
                valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
                break;
        default:
                valid = false;
        }


        if (valid) {
                env->insn_aux_data[insn_idx].ctx_field_size =
                        info.ctx_field_size;
                return 0;
        }

        verbose(env, "R%d invalid %s access off=%d size=%d\n",
                regno, reg_type_str(env, reg->type), off, size);

        return -EACCES;
}

static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
{
        return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
}

static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
{
        const struct bpf_reg_state *reg = reg_state(env, regno);

        return reg->type == PTR_TO_CTX;
}

static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
{
        const struct bpf_reg_state *reg = reg_state(env, regno);

        return type_is_sk_pointer(reg->type);
}

static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
{
        const struct bpf_reg_state *reg = reg_state(env, regno);

        return type_is_pkt_pointer(reg->type);
}

static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
{
        const struct bpf_reg_state *reg = reg_state(env, regno);

        /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
        return reg->type == PTR_TO_FLOW_KEYS;
}

static bool is_arena_reg(struct bpf_verifier_env *env, int regno)
{
        const struct bpf_reg_state *reg = reg_state(env, regno);

        return reg->type == PTR_TO_ARENA;
}

static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
#ifdef CONFIG_NET
        [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
        [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
        [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
#endif
        [CONST_PTR_TO_MAP] = btf_bpf_map_id,
};

static bool is_trusted_reg(const struct bpf_reg_state *reg)
{
        /* A referenced register is always trusted. */
        if (reg->ref_obj_id)
                return true;

        /* Types listed in the reg2btf_ids are always trusted */
        if (reg2btf_ids[base_type(reg->type)] &&
            !bpf_type_has_unsafe_modifiers(reg->type))
                return true;

        /* If a register is not referenced, it is trusted if it has the
         * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the
         * other type modifiers may be safe, but we elect to take an opt-in
         * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are
         * not.
         *
         * Eventually, we should make PTR_TRUSTED the single source of truth
         * for whether a register is trusted.
         */
        return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS &&
               !bpf_type_has_unsafe_modifiers(reg->type);
}

static bool is_rcu_reg(const struct bpf_reg_state *reg)
{
        return reg->type & MEM_RCU;
}

static void clear_trusted_flags(enum bpf_type_flag *flag)
{
        *flag &= ~(BPF_REG_TRUSTED_MODIFIERS | MEM_RCU);
}

static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
                                   const struct bpf_reg_state *reg,
                                   int off, int size, bool strict)
{
        struct tnum reg_off;
        int ip_align;

        /* Byte size accesses are always allowed. */
        if (!strict || size == 1)
                return 0;

        /* For platforms that do not have a Kconfig enabling
         * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
         * NET_IP_ALIGN is universally set to '2'.  And on platforms
         * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
         * to this code only in strict mode where we want to emulate
         * the NET_IP_ALIGN==2 checking.  Therefore use an
         * unconditional IP align value of '2'.
         */
        ip_align = 2;

        reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
        if (!tnum_is_aligned(reg_off, size)) {
                char tn_buf[48];

                tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                verbose(env,
                        "misaligned packet access off %d+%s+%d+%d size %d\n",
                        ip_align, tn_buf, reg->off, off, size);
                return -EACCES;
        }

        return 0;
}

static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
                                       const struct bpf_reg_state *reg,
                                       const char *pointer_desc,
                                       int off, int size, bool strict)
{
        struct tnum reg_off;

        /* Byte size accesses are always allowed. */
        if (!strict || size == 1)
                return 0;

        reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
        if (!tnum_is_aligned(reg_off, size)) {
                char tn_buf[48];

                tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
                        pointer_desc, tn_buf, reg->off, off, size);
                return -EACCES;
        }

        return 0;
}

static int check_ptr_alignment(struct bpf_verifier_env *env,
                               const struct bpf_reg_state *reg, int off,
                               int size, bool strict_alignment_once)
{
        bool strict = env->strict_alignment || strict_alignment_once;
        const char *pointer_desc = "";

        switch (reg->type) {
        case PTR_TO_PACKET:
        case PTR_TO_PACKET_META:
                /* Special case, because of NET_IP_ALIGN. Given metadata sits
                 * right in front, treat it the very same way.
                 */
                return check_pkt_ptr_alignment(env, reg, off, size, strict);
        case PTR_TO_FLOW_KEYS:
                pointer_desc = "flow keys ";
                break;
        case PTR_TO_MAP_KEY:
                pointer_desc = "key ";
                break;
        case PTR_TO_MAP_VALUE:
                pointer_desc = "value ";
                break;
        case PTR_TO_CTX:
                pointer_desc = "context ";
                break;
        case PTR_TO_STACK:
                pointer_desc = "stack ";
                /* The stack spill tracking logic in check_stack_write_fixed_off()
                 * and check_stack_read_fixed_off() relies on stack accesses being
                 * aligned.
                 */
                strict = true;
                break;
        case PTR_TO_SOCKET:
                pointer_desc = "sock ";
                break;
        case PTR_TO_SOCK_COMMON:
                pointer_desc = "sock_common ";
                break;
        case PTR_TO_TCP_SOCK:
                pointer_desc = "tcp_sock ";
                break;
        case PTR_TO_XDP_SOCK:
                pointer_desc = "xdp_sock ";
                break;
        case PTR_TO_ARENA:
                return 0;
        default:
                break;
        }
        return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
                                           strict);
}

static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog)
{
        if (!bpf_jit_supports_private_stack())
                return NO_PRIV_STACK;

        /* bpf_prog_check_recur() checks all prog types that use bpf trampoline
         * while kprobe/tp/perf_event/raw_tp don't use trampoline hence checked
         * explicitly.
         */
        switch (prog->type) {
        case BPF_PROG_TYPE_KPROBE:
        case BPF_PROG_TYPE_TRACEPOINT:
        case BPF_PROG_TYPE_PERF_EVENT:
        case BPF_PROG_TYPE_RAW_TRACEPOINT:
                return PRIV_STACK_ADAPTIVE;
        case BPF_PROG_TYPE_TRACING:
        case BPF_PROG_TYPE_LSM:
        case BPF_PROG_TYPE_STRUCT_OPS:
                if (prog->aux->priv_stack_requested || bpf_prog_check_recur(prog))
                        return PRIV_STACK_ADAPTIVE;
                fallthrough;
        default:
                break;
        }

        return NO_PRIV_STACK;
}

static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
{
        if (env->prog->jit_requested)
                return round_up(stack_depth, 16);

        /* round up to 32-bytes, since this is granularity
         * of interpreter stack size
         */
        return round_up(max_t(u32, stack_depth, 1), 32);
}

/* starting from main bpf function walk all instructions of the function
 * and recursively walk all callees that given function can call.
 * Ignore jump and exit insns.
 * Since recursion is prevented by check_cfg() this algorithm
 * only needs a local stack of MAX_CALL_FRAMES to remember callsites
 */
static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
                                         bool priv_stack_supported)
{
        struct bpf_subprog_info *subprog = env->subprog_info;
        struct bpf_insn *insn = env->prog->insnsi;
        int depth = 0, frame = 0, i, subprog_end, subprog_depth;
        bool tail_call_reachable = false;
        int ret_insn[MAX_CALL_FRAMES];
        int ret_prog[MAX_CALL_FRAMES];
        int j;

        i = subprog[idx].start;
        if (!priv_stack_supported)
                subprog[idx].priv_stack_mode = NO_PRIV_STACK;
process_func:
        /* protect against potential stack overflow that might happen when
         * bpf2bpf calls get combined with tailcalls. Limit the caller's stack
         * depth for such case down to 256 so that the worst case scenario
         * would result in 8k stack size (32 which is tailcall limit * 256 =
         * 8k).
         *
         * To get the idea what might happen, see an example:
         * func1 -> sub rsp, 128
         *  subfunc1 -> sub rsp, 256
         *  tailcall1 -> add rsp, 256
         *   func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320)
         *   subfunc2 -> sub rsp, 64
         *   subfunc22 -> sub rsp, 128
         *   tailcall2 -> add rsp, 128
         *    func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416)
         *
         * tailcall will unwind the current stack frame but it will not get rid
         * of caller's stack as shown on the example above.
         */
        if (idx && subprog[idx].has_tail_call && depth >= 256) {
                verbose(env,
                        "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n",
                        depth);
                return -EACCES;
        }

        subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth);
        if (priv_stack_supported) {
                /* Request private stack support only if the subprog stack
                 * depth is no less than BPF_PRIV_STACK_MIN_SIZE. This is to
                 * avoid jit penalty if the stack usage is small.
                 */
                if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN &&
                    subprog_depth >= BPF_PRIV_STACK_MIN_SIZE)
                        subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE;
        }

        if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
                if (subprog_depth > MAX_BPF_STACK) {
                        verbose(env, "stack size of subprog %d is %d. Too large\n",
                                idx, subprog_depth);
                        return -EACCES;
                }
        } else {
                depth += subprog_depth;
                if (depth > MAX_BPF_STACK) {
                        verbose(env, "combined stack size of %d calls is %d. Too large\n",
                                frame + 1, depth);
                        return -EACCES;
                }
        }
continue_func:
        subprog_end = subprog[idx + 1].start;
        for (; i < subprog_end; i++) {
                int next_insn, sidx;

                if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) {
                        bool err = false;

                        if (!is_bpf_throw_kfunc(insn + i))
                                continue;
                        if (subprog[idx].is_cb)
                                err = true;
                        for (int c = 0; c < frame && !err; c++) {
                                if (subprog[ret_prog[c]].is_cb) {
                                        err = true;
                                        break;
                                }
                        }
                        if (!err)
                                continue;
                        verbose(env,
                                "bpf_throw kfunc (insn %d) cannot be called from callback subprog %d\n",
                                i, idx);
                        return -EINVAL;
                }

                if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
                        continue;
                /* remember insn and function to return to */
                ret_insn[frame] = i + 1;
                ret_prog[frame] = idx;

                /* find the callee */
                next_insn = i + insn[i].imm + 1;
                sidx = find_subprog(env, next_insn);
                if (sidx < 0) {
                        WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
                                  next_insn);
                        return -EFAULT;
                }
                if (subprog[sidx].is_async_cb) {
                        if (subprog[sidx].has_tail_call) {
                                verbose(env, "verifier bug. subprog has tail_call and async cb\n");
                                return -EFAULT;
                        }
                        /* async callbacks don't increase bpf prog stack size unless called directly */
                        if (!bpf_pseudo_call(insn + i))
                                continue;
                        if (subprog[sidx].is_exception_cb) {
                                verbose(env, "insn %d cannot call exception cb directly\n", i);
                                return -EINVAL;
                        }
                }
                i = next_insn;
                idx = sidx;
                if (!priv_stack_supported)
                        subprog[idx].priv_stack_mode = NO_PRIV_STACK;

                if (subprog[idx].has_tail_call)
                        tail_call_reachable = true;

                frame++;
                if (frame >= MAX_CALL_FRAMES) {
                        verbose(env, "the call stack of %d frames is too deep !\n",
                                frame);
                        return -E2BIG;
                }
                goto process_func;
        }
        /* if tail call got detected across bpf2bpf calls then mark each of the
         * currently present subprog frames as tail call reachable subprogs;
         * this info will be utilized by JIT so that we will be preserving the
         * tail call counter throughout bpf2bpf calls combined with tailcalls
         */
        if (tail_call_reachable)
                for (j = 0; j < frame; j++) {
                        if (subprog[ret_prog[j]].is_exception_cb) {
                                verbose(env, "cannot tail call within exception cb\n");
                                return -EINVAL;
                        }
                        subprog[ret_prog[j]].tail_call_reachable = true;
                }
        if (subprog[0].tail_call_reachable)
                env->prog->aux->tail_call_reachable = true;

        /* end of for() loop means the last insn of the 'subprog'
         * was reached. Doesn't matter whether it was JA or EXIT
         */
        if (frame == 0)
                return 0;
        if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE)
                depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
        frame--;
        i = ret_insn[frame];
        idx = ret_prog[frame];
        goto continue_func;
}

static int check_max_stack_depth(struct bpf_verifier_env *env)
{
        enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN;
        struct bpf_subprog_info *si = env->subprog_info;
        bool priv_stack_supported;
        int ret;

        for (int i = 0; i < env->subprog_cnt; i++) {
                if (si[i].has_tail_call) {
                        priv_stack_mode = NO_PRIV_STACK;
                        break;
                }
        }

        if (priv_stack_mode == PRIV_STACK_UNKNOWN)
                priv_stack_mode = bpf_enable_priv_stack(env->prog);

        /* All async_cb subprogs use normal kernel stack. If a particular
         * subprog appears in both main prog and async_cb subtree, that
         * subprog will use normal kernel stack to avoid potential nesting.
         * The reverse subprog traversal ensures when main prog subtree is
         * checked, the subprogs appearing in async_cb subtrees are already
         * marked as using normal kernel stack, so stack size checking can
         * be done properly.
         */
        for (int i = env->subprog_cnt - 1; i >= 0; i--) {
                if (!i || si[i].is_async_cb) {
                        priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE;
                        ret = check_max_stack_depth_subprog(env, i, priv_stack_supported);
                        if (ret < 0)
                                return ret;
                }
        }

        for (int i = 0; i < env->subprog_cnt; i++) {
                if (si[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
                        env->prog->aux->jits_use_priv_stack = true;
                        break;
                }
        }

        return 0;
}

#ifndef CONFIG_BPF_JIT_ALWAYS_ON
static int get_callee_stack_depth(struct bpf_verifier_env *env,
                                  const struct bpf_insn *insn, int idx)
{
        int start = idx + insn->imm + 1, subprog;

        subprog = find_subprog(env, start);
        if (subprog < 0) {
                WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
                          start);
                return -EFAULT;
        }
        return env->subprog_info[subprog].stack_depth;
}
#endif

static int __check_buffer_access(struct bpf_verifier_env *env,
                                 const char *buf_info,
                                 const struct bpf_reg_state *reg,
                                 int regno, int off, int size)
{
        if (off < 0) {
                verbose(env,
                        "R%d invalid %s buffer access: off=%d, size=%d\n",
                        regno, buf_info, off, size);
                return -EACCES;
        }
        if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
                char tn_buf[48];

                tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                verbose(env,
                        "R%d invalid variable buffer offset: off=%d, var_off=%s\n",
                        regno, off, tn_buf);
                return -EACCES;
        }

        return 0;
}

static int check_tp_buffer_access(struct bpf_verifier_env *env,
                                  const struct bpf_reg_state *reg,
                                  int regno, int off, int size)
{
        int err;

        err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
        if (err)
                return err;

        if (off + size > env->prog->aux->max_tp_access)
                env->prog->aux->max_tp_access = off + size;

        return 0;
}

static int check_buffer_access(struct bpf_verifier_env *env,
                               const struct bpf_reg_state *reg,
                               int regno, int off, int size,
                               bool zero_size_allowed,
                               u32 *max_access)
{
        const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
        int err;

        err = __check_buffer_access(env, buf_info, reg, regno, off, size);
        if (err)
                return err;

        if (off + size > *max_access)
                *max_access = off + size;

        return 0;
}

/* BPF architecture zero extends alu32 ops into 64-bit registesr */
static void zext_32_to_64(struct bpf_reg_state *reg)
{
        reg->var_off = tnum_subreg(reg->var_off);
        __reg_assign_32_into_64(reg);
}

/* truncate register to smaller size (in bytes)
 * must be called with size < BPF_REG_SIZE
 */
static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
{
        u64 mask;

        /* clear high bits in bit representation */
        reg->var_off = tnum_cast(reg->var_off, size);

        /* fix arithmetic bounds */
        mask = ((u64)1 << (size * 8)) - 1;
        if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
                reg->umin_value &= mask;
                reg->umax_value &= mask;
        } else {
                reg->umin_value = 0;
                reg->umax_value = mask;
        }
        reg->smin_value = reg->umin_value;
        reg->smax_value = reg->umax_value;

        /* If size is smaller than 32bit register the 32bit register
         * values are also truncated so we push 64-bit bounds into
         * 32-bit bounds. Above were truncated < 32-bits already.
         */
        if (size < 4)
                __mark_reg32_unbounded(reg);

        reg_bounds_sync(reg);
}

static void set_sext64_default_val(struct bpf_reg_state *reg, int size)
{
        if (size == 1) {
                reg->smin_value = reg->s32_min_value = S8_MIN;
                reg->smax_value = reg->s32_max_value = S8_MAX;
        } else if (size == 2) {
                reg->smin_value = reg->s32_min_value = S16_MIN;
                reg->smax_value = reg->s32_max_value = S16_MAX;
        } else {
                /* size == 4 */
                reg->smin_value = reg->s32_min_value = S32_MIN;
                reg->smax_value = reg->s32_max_value = S32_MAX;
        }
        reg->umin_value = reg->u32_min_value = 0;
        reg->umax_value = U64_MAX;
        reg->u32_max_value = U32_MAX;
        reg->var_off = tnum_unknown;
}

static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size)
{
        s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval;
        u64 top_smax_value, top_smin_value;
        u64 num_bits = size * 8;

        if (tnum_is_const(reg->var_off)) {
                u64_cval = reg->var_off.value;
                if (size == 1)
                        reg->var_off = tnum_const((s8)u64_cval);
                else if (size == 2)
                        reg->var_off = tnum_const((s16)u64_cval);
                else
                        /* size == 4 */
                        reg->var_off = tnum_const((s32)u64_cval);

                u64_cval = reg->var_off.value;
                reg->smax_value = reg->smin_value = u64_cval;
                reg->umax_value = reg->umin_value = u64_cval;
                reg->s32_max_value = reg->s32_min_value = u64_cval;
                reg->u32_max_value = reg->u32_min_value = u64_cval;
                return;
        }

        top_smax_value = ((u64)reg->smax_value >> num_bits) << num_bits;
        top_smin_value = ((u64)reg->smin_value >> num_bits) << num_bits;

        if (top_smax_value != top_smin_value)
                goto out;

        /* find the s64_min and s64_min after sign extension */
        if (size == 1) {
                init_s64_max = (s8)reg->smax_value;
                init_s64_min = (s8)reg->smin_value;
        } else if (size == 2) {
                init_s64_max = (s16)reg->smax_value;
                init_s64_min = (s16)reg->smin_value;
        } else {
                init_s64_max = (s32)reg->smax_value;
                init_s64_min = (s32)reg->smin_value;
        }

        s64_max = max(init_s64_max, init_s64_min);
        s64_min = min(init_s64_max, init_s64_min);

        /* both of s64_max/s64_min positive or negative */
        if ((s64_max >= 0) == (s64_min >= 0)) {
                reg->s32_min_value = reg->smin_value = s64_min;
                reg->s32_max_value = reg->smax_value = s64_max;
                reg->u32_min_value = reg->umin_value = s64_min;
                reg->u32_max_value = reg->umax_value = s64_max;
                reg->var_off = tnum_range(s64_min, s64_max);
                return;
        }

out:
        set_sext64_default_val(reg, size);
}

static void set_sext32_default_val(struct bpf_reg_state *reg, int size)
{
        if (size == 1) {
                reg->s32_min_value = S8_MIN;
                reg->s32_max_value = S8_MAX;
        } else {
                /* size == 2 */
                reg->s32_min_value = S16_MIN;
                reg->s32_max_value = S16_MAX;
        }
        reg->u32_min_value = 0;
        reg->u32_max_value = U32_MAX;
        reg->var_off = tnum_subreg(tnum_unknown);
}

static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size)
{
        s32 init_s32_max, init_s32_min, s32_max, s32_min, u32_val;
        u32 top_smax_value, top_smin_value;
        u32 num_bits = size * 8;

        if (tnum_is_const(reg->var_off)) {
                u32_val = reg->var_off.value;
                if (size == 1)
                        reg->var_off = tnum_const((s8)u32_val);
                else
                        reg->var_off = tnum_const((s16)u32_val);

                u32_val = reg->var_off.value;
                reg->s32_min_value = reg->s32_max_value = u32_val;
                reg->u32_min_value = reg->u32_max_value = u32_val;
                return;
        }

        top_smax_value = ((u32)reg->s32_max_value >> num_bits) << num_bits;
        top_smin_value = ((u32)reg->s32_min_value >> num_bits) << num_bits;

        if (top_smax_value != top_smin_value)
                goto out;

        /* find the s32_min and s32_min after sign extension */
        if (size == 1) {
                init_s32_max = (s8)reg->s32_max_value;
                init_s32_min = (s8)reg->s32_min_value;
        } else {
                /* size == 2 */
                init_s32_max = (s16)reg->s32_max_value;
                init_s32_min = (s16)reg->s32_min_value;
        }
        s32_max = max(init_s32_max, init_s32_min);
        s32_min = min(init_s32_max, init_s32_min);

        if ((s32_min >= 0) == (s32_max >= 0)) {
                reg->s32_min_value = s32_min;
                reg->s32_max_value = s32_max;
                reg->u32_min_value = (u32)s32_min;
                reg->u32_max_value = (u32)s32_max;
                reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max));
                return;
        }

out:
        set_sext32_default_val(reg, size);
}

static bool bpf_map_is_rdonly(const struct bpf_map *map)
{
        /* A map is considered read-only if the following condition are true:
         *
         * 1) BPF program side cannot change any of the map content. The
         *    BPF_F_RDONLY_PROG flag is throughout the lifetime of a map
         *    and was set at map creation time.
         * 2) The map value(s) have been initialized from user space by a
         *    loader and then "frozen", such that no new map update/delete
         *    operations from syscall side are possible for the rest of
         *    the map's lifetime from that point onwards.
         * 3) Any parallel/pending map update/delete operations from syscall
         *    side have been completed. Only after that point, it's safe to
         *    assume that map value(s) are immutable.
         */
        return (map->map_flags & BPF_F_RDONLY_PROG) &&
               READ_ONCE(map->frozen) &&
               !bpf_map_write_active(map);
}

static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
                               bool is_ldsx)
{
        void *ptr;
        u64 addr;
        int err;

        err = map->ops->map_direct_value_addr(map, &addr, off);
        if (err)
                return err;
        ptr = (void *)(long)addr + off;

        switch (size) {
        case sizeof(u8):
                *val = is_ldsx ? (s64)*(s8 *)ptr : (u64)*(u8 *)ptr;
                break;
        case sizeof(u16):
                *val = is_ldsx ? (s64)*(s16 *)ptr : (u64)*(u16 *)ptr;
                break;
        case sizeof(u32):
                *val = is_ldsx ? (s64)*(s32 *)ptr : (u64)*(u32 *)ptr;
                break;
        case sizeof(u64):
                *val = *(u64 *)ptr;
                break;
        default:
                return -EINVAL;
        }
        return 0;
}

#define BTF_TYPE_SAFE_RCU(__type)  __PASTE(__type, __safe_rcu)
#define BTF_TYPE_SAFE_RCU_OR_NULL(__type)  __PASTE(__type, __safe_rcu_or_null)
#define BTF_TYPE_SAFE_TRUSTED(__type)  __PASTE(__type, __safe_trusted)
#define BTF_TYPE_SAFE_TRUSTED_OR_NULL(__type)  __PASTE(__type, __safe_trusted_or_null)

/*
 * Allow list few fields as RCU trusted or full trusted.
 * This logic doesn't allow mix tagging and will be removed once GCC supports
 * btf_type_tag.
 */

/* RCU trusted: these fields are trusted in RCU CS and never NULL */
BTF_TYPE_SAFE_RCU(struct task_struct) {
        const cpumask_t *cpus_ptr;
        struct css_set __rcu *cgroups;
        struct task_struct __rcu *real_parent;
        struct task_struct *group_leader;
};

BTF_TYPE_SAFE_RCU(struct cgroup) {
        /* cgrp->kn is always accessible as documented in kernel/cgroup/cgroup.c */
        struct kernfs_node *kn;
};

BTF_TYPE_SAFE_RCU(struct css_set) {
        struct cgroup *dfl_cgrp;
};

/* RCU trusted: these fields are trusted in RCU CS and can be NULL */
BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct) {
        struct file __rcu *exe_file;
};

/* skb->sk, req->sk are not RCU protected, but we mark them as such
 * because bpf prog accessible sockets are SOCK_RCU_FREE.
 */
BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff) {
        struct sock *sk;
};

BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock) {
        struct sock *sk;
};

/* full trusted: these fields are trusted even outside of RCU CS and never NULL */
BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta) {
        struct seq_file *seq;
};

BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task) {
        struct bpf_iter_meta *meta;
        struct task_struct *task;
};

BTF_TYPE_SAFE_TRUSTED(struct linux_binprm) {
        struct file *file;
};

BTF_TYPE_SAFE_TRUSTED(struct file) {
        struct inode *f_inode;
};

BTF_TYPE_SAFE_TRUSTED(struct dentry) {
        /* no negative dentry-s in places where bpf can see it */
        struct inode *d_inode;
};

BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) {
        struct sock *sk;
};

static bool type_is_rcu(struct bpf_verifier_env *env,
                        struct bpf_reg_state *reg,
                        const char *field_name, u32 btf_id)
{
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct task_struct));
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup));
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct css_set));

        return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu");
}

static bool type_is_rcu_or_null(struct bpf_verifier_env *env,
                                struct bpf_reg_state *reg,
                                const char *field_name, u32 btf_id)
{
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct));
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff));
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock));

        return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null");
}

static bool type_is_trusted(struct bpf_verifier_env *env,
                            struct bpf_reg_state *reg,
                            const char *field_name, u32 btf_id)
{
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta));
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task));
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm));
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file));
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct dentry));

        return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");
}

static bool type_is_trusted_or_null(struct bpf_verifier_env *env,
                                    struct bpf_reg_state *reg,
                                    const char *field_name, u32 btf_id)
{
        BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket));

        return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id,
                                          "__safe_trusted_or_null");
}

static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
                                   struct bpf_reg_state *regs,
                                   int regno, int off, int size,
                                   enum bpf_access_type atype,
                                   int value_regno)
{
        struct bpf_reg_state *reg = regs + regno;
        const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
        const char *tname = btf_name_by_offset(reg->btf, t->name_off);
        const char *field_name = NULL;
        enum bpf_type_flag flag = 0;
        u32 btf_id = 0;
        int ret;

        if (!env->allow_ptr_leaks) {
                verbose(env,
                        "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
                        tname);
                return -EPERM;
        }
        if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) {
                verbose(env,
                        "Cannot access kernel 'struct %s' from non-GPL compatible program\n",
                        tname);
                return -EINVAL;
        }
        if (off < 0) {
                verbose(env,
                        "R%d is ptr_%s invalid negative access: off=%d\n",
                        regno, tname, off);
                return -EACCES;
        }
        if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
                char tn_buf[48];

                tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                verbose(env,
                        "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
                        regno, tname, off, tn_buf);
                return -EACCES;
        }

        if (reg->type & MEM_USER) {
                verbose(env,
                        "R%d is ptr_%s access user memory: off=%d\n",
                        regno, tname, off);
                return -EACCES;
        }

        if (reg->type & MEM_PERCPU) {
                verbose(env,
                        "R%d is ptr_%s access percpu memory: off=%d\n",
                        regno, tname, off);
                return -EACCES;
        }

        if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {
                if (!btf_is_kernel(reg->btf)) {
                        verbose(env, "verifier internal error: reg->btf must be kernel btf\n");
                        return -EFAULT;
                }
                ret = env->ops->btf_struct_access(&env->log, reg, off, size);
        } else {
                /* Writes are permitted with default btf_struct_access for
                 * program allocated objects (which always have ref_obj_id > 0),
                 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC.
                 */
                if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) {
                        verbose(env, "only read is supported\n");
                        return -EACCES;
                }

                if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) &&
                    !(reg->type & MEM_RCU) && !reg->ref_obj_id) {
                        verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n");
                        return -EFAULT;
                }

                ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name);
        }

        if (ret < 0)
                return ret;

        if (ret != PTR_TO_BTF_ID) {
                /* just mark; */

        } else if (type_flag(reg->type) & PTR_UNTRUSTED) {
                /* If this is an untrusted pointer, all pointers formed by walking it
                 * also inherit the untrusted flag.
                 */
                flag = PTR_UNTRUSTED;

        } else if (is_trusted_reg(reg) || is_rcu_reg(reg)) {
                /* By default any pointer obtained from walking a trusted pointer is no
                 * longer trusted, unless the field being accessed has explicitly been
                 * marked as inheriting its parent's state of trust (either full or RCU).
                 * For example:
                 * 'cgroups' pointer is untrusted if task->cgroups dereference
                 * happened in a sleepable program outside of bpf_rcu_read_lock()
                 * section. In a non-sleepable program it's trusted while in RCU CS (aka MEM_RCU).
                 * Note bpf_rcu_read_unlock() converts MEM_RCU pointers to PTR_UNTRUSTED.
                 *
                 * A regular RCU-protected pointer with __rcu tag can also be deemed
                 * trusted if we are in an RCU CS. Such pointer can be NULL.
                 */
                if (type_is_trusted(env, reg, field_name, btf_id)) {
                        flag |= PTR_TRUSTED;
                } else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) {
                        flag |= PTR_TRUSTED | PTR_MAYBE_NULL;
                } else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) {
                        if (type_is_rcu(env, reg, field_name, btf_id)) {
                                /* ignore __rcu tag and mark it MEM_RCU */
                                flag |= MEM_RCU;
                        } else if (flag & MEM_RCU ||
                                   type_is_rcu_or_null(env, reg, field_name, btf_id)) {
                                /* __rcu tagged pointers can be NULL */
                                flag |= MEM_RCU | PTR_MAYBE_NULL;

                                /* We always trust them */
                                if (type_is_rcu_or_null(env, reg, field_name, btf_id) &&
                                    flag & PTR_UNTRUSTED)
                                        flag &= ~PTR_UNTRUSTED;
                        } else if (flag & (MEM_PERCPU | MEM_USER)) {
                                /* keep as-is */
                        } else {
                                /* walking unknown pointers yields old deprecated PTR_TO_BTF_ID */
                                clear_trusted_flags(&flag);
                        }
                } else {
                        /*
                         * If not in RCU CS or MEM_RCU pointer can be NULL then
                         * aggressively mark as untrusted otherwise such
                         * pointers will be plain PTR_TO_BTF_ID without flags
                         * and will be allowed to be passed into helpers for
                         * compat reasons.
                         */
                        flag = PTR_UNTRUSTED;
                }
        } else {
                /* Old compat. Deprecated */
                clear_trusted_flags(&flag);
        }

        if (atype == BPF_READ && value_regno >= 0)
                mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);

        return 0;
}

static int check_ptr_to_map_access(struct bpf_verifier_env *env,
                                   struct bpf_reg_state *regs,
                                   int regno, int off, int size,
                                   enum bpf_access_type atype,
                                   int value_regno)
{
        struct bpf_reg_state *reg = regs + regno;
        struct bpf_map *map = reg->map_ptr;
        struct bpf_reg_state map_reg;
        enum bpf_type_flag flag = 0;
        const struct btf_type *t;
        const char *tname;
        u32 btf_id;
        int ret;

        if (!btf_vmlinux) {
                verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");
                return -ENOTSUPP;
        }

        if (!map->ops->map_btf_id || !*map->ops->map_btf_id) {
                verbose(env, "map_ptr access not supported for map type %d\n",
                        map->map_type);
                return -ENOTSUPP;
        }

        t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id);
        tname = btf_name_by_offset(btf_vmlinux, t->name_off);

        if (!env->allow_ptr_leaks) {
                verbose(env,
                        "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
                        tname);
                return -EPERM;
        }

        if (off < 0) {
                verbose(env, "R%d is %s invalid negative access: off=%d\n",
                        regno, tname, off);
                return -EACCES;
        }

        if (atype != BPF_READ) {
                verbose(env, "only read from %s is supported\n", tname);
                return -EACCES;
        }

        /* Simulate access to a PTR_TO_BTF_ID */
        memset(&map_reg, 0, sizeof(map_reg));
        mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0);
        ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL);
        if (ret < 0)
                return ret;

        if (value_regno >= 0)
                mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag);

        return 0;
}

/* Check that the stack access at the given offset is within bounds. The
 * maximum valid offset is -1.
 *
 * The minimum valid offset is -MAX_BPF_STACK for writes, and
 * -state->allocated_stack for reads.
 */
static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
                                          s64 off,
                                          struct bpf_func_state *state,
                                          enum bpf_access_type t)
{
        int min_valid_off;

        if (t == BPF_WRITE || env->allow_uninit_stack)
                min_valid_off = -MAX_BPF_STACK;
        else
                min_valid_off = -state->allocated_stack;

        if (off < min_valid_off || off > -1)
                return -EACCES;
        return 0;
}

/* Check that the stack access at 'regno + off' falls within the maximum stack
 * bounds.
 *
 * 'off' includes `regno->offset`, but not its dynamic part (if any).
 */
static int check_stack_access_within_bounds(
                struct bpf_verifier_env *env,
                int regno, int off, int access_size,
                enum bpf_access_type type)
{
        struct bpf_reg_state *regs = cur_regs(env);
        struct bpf_reg_state *reg = regs + regno;
        struct bpf_func_state *state = func(env, reg);
        s64 min_off, max_off;
        int err;
        char *err_extra;

        if (type == BPF_READ)
                err_extra = " read from";
        else
                err_extra = " write to";

        if (tnum_is_const(reg->var_off)) {
                min_off = (s64)reg->var_off.value + off;
                max_off = min_off + access_size;
        } else {
                if (reg->smax_value >= BPF_MAX_VAR_OFF ||
                    reg->smin_value <= -BPF_MAX_VAR_OFF) {
                        verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
                                err_extra, regno);
                        return -EACCES;
                }
                min_off = reg->smin_value + off;
                max_off = reg->smax_value + off + access_size;
        }

        err = check_stack_slot_within_bounds(env, min_off, state, type);
        if (!err && max_off > 0)
                err = -EINVAL; /* out of stack access into non-negative offsets */
        if (!err && access_size < 0)
                /* access_size should not be negative (or overflow an int); others checks
                 * along the way should have prevented such an access.
                 */
                err = -EFAULT; /* invalid negative access size; integer overflow? */

        if (err) {
                if (tnum_is_const(reg->var_off)) {
                        verbose(env, "invalid%s stack R%d off=%d size=%d\n",
                                err_extra, regno, off, access_size);
                } else {
                        char tn_buf[48];

                        tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                        verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n",
                                err_extra, regno, tn_buf, off, access_size);
                }
                return err;
        }

        /* Note that there is no stack access with offset zero, so the needed stack
         * size is -min_off, not -min_off+1.
         */
        return grow_stack_state(env, state, -min_off /* size */);
}

static bool get_func_retval_range(struct bpf_prog *prog,
                                  struct bpf_retval_range *range)
{
        if (prog->type == BPF_PROG_TYPE_LSM &&
                prog->expected_attach_type == BPF_LSM_MAC &&
                !bpf_lsm_get_retval_range(prog, range)) {
                return true;
        }
        return false;
}

/* check whether memory at (regno + off) is accessible for t = (read | write)
 * if t==write, value_regno is a register which value is stored into memory
 * if t==read, value_regno is a register which will receive the value from memory
 * if t==write && value_regno==-1, some unknown value is stored into memory
 * if t==read && value_regno==-1, don't care what we read from memory
 */
static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
                            int off, int bpf_size, enum bpf_access_type t,
                            int value_regno, bool strict_alignment_once, bool is_ldsx)
{
        struct bpf_reg_state *regs = cur_regs(env);
        struct bpf_reg_state *reg = regs + regno;
        int size, err = 0;

        size = bpf_size_to_bytes(bpf_size);
        if (size < 0)
                return size;

        /* alignment checks will add in reg->off themselves */
        err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
        if (err)
                return err;

        /* for access checks, reg->off is just part of off */
        off += reg->off;

        if (reg->type == PTR_TO_MAP_KEY) {
                if (t == BPF_WRITE) {
                        verbose(env, "write to change key R%d not allowed\n", regno);
                        return -EACCES;
                }

                err = check_mem_region_access(env, regno, off, size,
                                              reg->map_ptr->key_size, false);
                if (err)
                        return err;
                if (value_regno >= 0)
                        mark_reg_unknown(env, regs, value_regno);
        } else if (reg->type == PTR_TO_MAP_VALUE) {
                struct btf_field *kptr_field = NULL;

                if (t == BPF_WRITE && value_regno >= 0 &&
                    is_pointer_value(env, value_regno)) {
                        verbose(env, "R%d leaks addr into map\n", value_regno);
                        return -EACCES;
                }
                err = check_map_access_type(env, regno, off, size, t);
                if (err)
                        return err;
                err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT);
                if (err)
                        return err;
                if (tnum_is_const(reg->var_off))
                        kptr_field = btf_record_find(reg->map_ptr->record,
                                                     off + reg->var_off.value, BPF_KPTR | BPF_UPTR);
                if (kptr_field) {
                        err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);
                } else if (t == BPF_READ && value_regno >= 0) {
                        struct bpf_map *map = reg->map_ptr;

                        /* if map is read-only, track its contents as scalars */
                        if (tnum_is_const(reg->var_off) &&
                            bpf_map_is_rdonly(map) &&
                            map->ops->map_direct_value_addr) {
                                int map_off = off + reg->var_off.value;
                                u64 val = 0;

                                err = bpf_map_direct_read(map, map_off, size,
                                                          &val, is_ldsx);
                                if (err)
                                        return err;

                                regs[value_regno].type = SCALAR_VALUE;
                                __mark_reg_known(&regs[value_regno], val);
                        } else {
                                mark_reg_unknown(env, regs, value_regno);
                        }
                }
        } else if (base_type(reg->type) == PTR_TO_MEM) {
                bool rdonly_mem = type_is_rdonly_mem(reg->type);

                if (type_may_be_null(reg->type)) {
                        verbose(env, "R%d invalid mem access '%s'\n", regno,
                                reg_type_str(env, reg->type));
                        return -EACCES;
                }

                if (t == BPF_WRITE && rdonly_mem) {
                        verbose(env, "R%d cannot write into %s\n",
                                regno, reg_type_str(env, reg->type));
                        return -EACCES;
                }

                if (t == BPF_WRITE && value_regno >= 0 &&
                    is_pointer_value(env, value_regno)) {
                        verbose(env, "R%d leaks addr into mem\n", value_regno);
                        return -EACCES;
                }

                err = check_mem_region_access(env, regno, off, size,
                                              reg->mem_size, false);
                if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem))
                        mark_reg_unknown(env, regs, value_regno);
        } else if (reg->type == PTR_TO_CTX) {
                bool is_retval = false;
                struct bpf_retval_range range;
                enum bpf_reg_type reg_type = SCALAR_VALUE;
                struct btf *btf = NULL;
                u32 btf_id = 0;

                if (t == BPF_WRITE && value_regno >= 0 &&
                    is_pointer_value(env, value_regno)) {
                        verbose(env, "R%d leaks addr into ctx\n", value_regno);
                        return -EACCES;
                }

                err = check_ptr_off_reg(env, reg, regno);
                if (err < 0)
                        return err;

                err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf,
                                       &btf_id, &is_retval, is_ldsx);
                if (err)
                        verbose_linfo(env, insn_idx, "; ");
                if (!err && t == BPF_READ && value_regno >= 0) {
                        /* ctx access returns either a scalar, or a
                         * PTR_TO_PACKET[_META,_END]. In the latter
                         * case, we know the offset is zero.
                         */
                        if (reg_type == SCALAR_VALUE) {
                                if (is_retval && get_func_retval_range(env->prog, &range)) {
                                        err = __mark_reg_s32_range(env, regs, value_regno,
                                                                   range.minval, range.maxval);
                                        if (err)
                                                return err;
                                } else {
                                        mark_reg_unknown(env, regs, value_regno);
                                }
                        } else {
                                mark_reg_known_zero(env, regs,
                                                    value_regno);
                                if (type_may_be_null(reg_type))
                                        regs[value_regno].id = ++env->id_gen;
                                /* A load of ctx field could have different
                                 * actual load size with the one encoded in the
                                 * insn. When the dst is PTR, it is for sure not
                                 * a sub-register.
                                 */
                                regs[value_regno].subreg_def = DEF_NOT_SUBREG;
                                if (base_type(reg_type) == PTR_TO_BTF_ID) {
                                        regs[value_regno].btf = btf;
                                        regs[value_regno].btf_id = btf_id;
                                }
                        }
                        regs[value_regno].type = reg_type;
                }

        } else if (reg->type == PTR_TO_STACK) {
                /* Basic bounds checks. */
                err = check_stack_access_within_bounds(env, regno, off, size, t);
                if (err)
                        return err;

                if (t == BPF_READ)
                        err = check_stack_read(env, regno, off, size,
                                               value_regno);
                else
                        err = check_stack_write(env, regno, off, size,
                                                value_regno, insn_idx);
        } else if (reg_is_pkt_pointer(reg)) {
                if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
                        verbose(env, "cannot write into packet\n");
                        return -EACCES;
                }
                if (t == BPF_WRITE && value_regno >= 0 &&
                    is_pointer_value(env, value_regno)) {
                        verbose(env, "R%d leaks addr into packet\n",
                                value_regno);
                        return -EACCES;
                }
                err = check_packet_access(env, regno, off, size, false);
                if (!err && t == BPF_READ && value_regno >= 0)
                        mark_reg_unknown(env, regs, value_regno);
        } else if (reg->type == PTR_TO_FLOW_KEYS) {
                if (t == BPF_WRITE && value_regno >= 0 &&
                    is_pointer_value(env, value_regno)) {
                        verbose(env, "R%d leaks addr into flow keys\n",
                                value_regno);
                        return -EACCES;
                }

                err = check_flow_keys_access(env, off, size);
                if (!err && t == BPF_READ && value_regno >= 0)
                        mark_reg_unknown(env, regs, value_regno);
        } else if (type_is_sk_pointer(reg->type)) {
                if (t == BPF_WRITE) {
                        verbose(env, "R%d cannot write into %s\n",
                                regno, reg_type_str(env, reg->type));
                        return -EACCES;
                }
                err = check_sock_access(env, insn_idx, regno, off, size, t);
                if (!err && value_regno >= 0)
                        mark_reg_unknown(env, regs, value_regno);
        } else if (reg->type == PTR_TO_TP_BUFFER) {
                err = check_tp_buffer_access(env, reg, regno, off, size);
                if (!err && t == BPF_READ && value_regno >= 0)
                        mark_reg_unknown(env, regs, value_regno);
        } else if (base_type(reg->type) == PTR_TO_BTF_ID &&
                   !type_may_be_null(reg->type)) {
                err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
                                              value_regno);
        } else if (reg->type == CONST_PTR_TO_MAP) {
                err = check_ptr_to_map_access(env, regs, regno, off, size, t,
                                              value_regno);
        } else if (base_type(reg->type) == PTR_TO_BUF) {
                bool rdonly_mem = type_is_rdonly_mem(reg->type);
                u32 *max_access;

                if (rdonly_mem) {
                        if (t == BPF_WRITE) {
                                verbose(env, "R%d cannot write into %s\n",
                                        regno, reg_type_str(env, reg->type));
                                return -EACCES;
                        }
                        max_access = &env->prog->aux->max_rdonly_access;
                } else {
                        max_access = &env->prog->aux->max_rdwr_access;
                }

                err = check_buffer_access(env, reg, regno, off, size, false,
                                          max_access);

                if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ))
                        mark_reg_unknown(env, regs, value_regno);
        } else if (reg->type == PTR_TO_ARENA) {
                if (t == BPF_READ && value_regno >= 0)
                        mark_reg_unknown(env, regs, value_regno);
        } else {
                verbose(env, "R%d invalid mem access '%s'\n", regno,
                        reg_type_str(env, reg->type));
                return -EACCES;
        }

        if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
            regs[value_regno].type == SCALAR_VALUE) {
                if (!is_ldsx)
                        /* b/h/w load zero-extends, mark upper bits as known 0 */
                        coerce_reg_to_size(&regs[value_regno], size);
                else
                        coerce_reg_to_size_sx(&regs[value_regno], size);
        }
        return err;
}

static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
                             bool allow_trust_mismatch);

static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
{
        int load_reg;
        int err;

        switch (insn->imm) {
        case BPF_ADD:
        case BPF_ADD | BPF_FETCH:
        case BPF_AND:
        case BPF_AND | BPF_FETCH:
        case BPF_OR:
        case BPF_OR | BPF_FETCH:
        case BPF_XOR:
        case BPF_XOR | BPF_FETCH:
        case BPF_XCHG:
        case BPF_CMPXCHG:
                break;
        default:
                verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);
                return -EINVAL;
        }

        if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
                verbose(env, "invalid atomic operand size\n");
                return -EINVAL;
        }

        /* check src1 operand */
        err = check_reg_arg(env, insn->src_reg, SRC_OP);
        if (err)
                return err;

        /* check src2 operand */
        err = check_reg_arg(env, insn->dst_reg, SRC_OP);
        if (err)
                return err;

        if (insn->imm == BPF_CMPXCHG) {
                /* Check comparison of R0 with memory location */
                const u32 aux_reg = BPF_REG_0;

                err = check_reg_arg(env, aux_reg, SRC_OP);
                if (err)
                        return err;

                if (is_pointer_value(env, aux_reg)) {
                        verbose(env, "R%d leaks addr into mem\n", aux_reg);
                        return -EACCES;
                }
        }

        if (is_pointer_value(env, insn->src_reg)) {
                verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
                return -EACCES;
        }

        if (is_ctx_reg(env, insn->dst_reg) ||
            is_pkt_reg(env, insn->dst_reg) ||
            is_flow_key_reg(env, insn->dst_reg) ||
            is_sk_reg(env, insn->dst_reg) ||
            (is_arena_reg(env, insn->dst_reg) && !bpf_jit_supports_insn(insn, true))) {
                verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
                        insn->dst_reg,
                        reg_type_str(env, reg_state(env, insn->dst_reg)->type));
                return -EACCES;
        }

        if (insn->imm & BPF_FETCH) {
                if (insn->imm == BPF_CMPXCHG)
                        load_reg = BPF_REG_0;
                else
                        load_reg = insn->src_reg;

                /* check and record load of old value */
                err = check_reg_arg(env, load_reg, DST_OP);
                if (err)
                        return err;
        } else {
                /* This instruction accesses a memory location but doesn't
                 * actually load it into a register.
                 */
                load_reg = -1;
        }

        /* Check whether we can read the memory, with second call for fetch
         * case to simulate the register fill.
         */
        err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
                               BPF_SIZE(insn->code), BPF_READ, -1, true, false);
        if (!err && load_reg >= 0)
                err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
                                       BPF_SIZE(insn->code), BPF_READ, load_reg,
                                       true, false);
        if (err)
                return err;

        if (is_arena_reg(env, insn->dst_reg)) {
                err = save_aux_ptr_type(env, PTR_TO_ARENA, false);
                if (err)
                        return err;
        }
        /* Check whether we can write into the same memory. */
        err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
                               BPF_SIZE(insn->code), BPF_WRITE, -1, true, false);
        if (err)
                return err;
        return 0;
}

/* When register 'regno' is used to read the stack (either directly or through
 * a helper function) make sure that it's within stack boundary and, depending
 * on the access type and privileges, that all elements of the stack are
 * initialized.
 *
 * 'off' includes 'regno->off', but not its dynamic part (if any).
 *
 * All registers that have been spilled on the stack in the slots within the
 * read offsets are marked as read.
 */
static int check_stack_range_initialized(
                struct bpf_verifier_env *env, int regno, int off,
                int access_size, bool zero_size_allowed,
                enum bpf_access_type type, struct bpf_call_arg_meta *meta)
{
        struct bpf_reg_state *reg = reg_state(env, regno);
        struct bpf_func_state *state = func(env, reg);
        int err, min_off, max_off, i, j, slot, spi;
        /* Some accesses can write anything into the stack, others are
         * read-only.
         */
        bool clobber = false;

        if (access_size == 0 && !zero_size_allowed) {
                verbose(env, "invalid zero-sized read\n");
                return -EACCES;
        }

        if (type == BPF_WRITE)
                clobber = true;

        err = check_stack_access_within_bounds(env, regno, off, access_size, type);
        if (err)
                return err;


        if (tnum_is_const(reg->var_off)) {
                min_off = max_off = reg->var_off.value + off;
        } else {
                /* Variable offset is prohibited for unprivileged mode for
                 * simplicity since it requires corresponding support in
                 * Spectre masking for stack ALU.
                 * See also retrieve_ptr_limit().
                 */
                if (!env->bypass_spec_v1) {
                        char tn_buf[48];

                        tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                        verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",
                                regno, tn_buf);
                        return -EACCES;
                }
                /* Only initialized buffer on stack is allowed to be accessed
                 * with variable offset. With uninitialized buffer it's hard to
                 * guarantee that whole memory is marked as initialized on
                 * helper return since specific bounds are unknown what may
                 * cause uninitialized stack leaking.
                 */
                if (meta && meta->raw_mode)
                        meta = NULL;

                min_off = reg->smin_value + off;
                max_off = reg->smax_value + off;
        }

        if (meta && meta->raw_mode) {
                /* Ensure we won't be overwriting dynptrs when simulating byte
                 * by byte access in check_helper_call using meta.access_size.
                 * This would be a problem if we have a helper in the future
                 * which takes:
                 *
                 *        helper(uninit_mem, len, dynptr)
                 *
                 * Now, uninint_mem may overlap with dynptr pointer. Hence, it
                 * may end up writing to dynptr itself when touching memory from
                 * arg 1. This can be relaxed on a case by case basis for known
                 * safe cases, but reject due to the possibilitiy of aliasing by
                 * default.
                 */
                for (i = min_off; i < max_off + access_size; i++) {
                        int stack_off = -i - 1;

                        spi = __get_spi(i);
                        /* raw_mode may write past allocated_stack */
                        if (state->allocated_stack <= stack_off)
                                continue;
                        if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) {
                                verbose(env, "potential write to dynptr at off=%d disallowed\n", i);
                                return -EACCES;
                        }
                }
                meta->access_size = access_size;
                meta->regno = regno;
                return 0;
        }

        for (i = min_off; i < max_off + access_size; i++) {
                u8 *stype;

                slot = -i - 1;
                spi = slot / BPF_REG_SIZE;
                if (state->allocated_stack <= slot) {
                        verbose(env, "verifier bug: allocated_stack too small\n");
                        return -EFAULT;
                }

                stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
                if (*stype == STACK_MISC)
                        goto mark;
                if ((*stype == STACK_ZERO) ||
                    (*stype == STACK_INVALID && env->allow_uninit_stack)) {
                        if (clobber) {
                                /* helper can write anything into the stack */
                                *stype = STACK_MISC;
                        }
                        goto mark;
                }

                if (is_spilled_reg(&state->stack[spi]) &&
                    (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
                     env->allow_ptr_leaks)) {
                        if (clobber) {
                                __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
                                for (j = 0; j < BPF_REG_SIZE; j++)
                                        scrub_spilled_slot(&state->stack[spi].slot_type[j]);
                        }
                        goto mark;
                }

                if (tnum_is_const(reg->var_off)) {
                        verbose(env, "invalid read from stack R%d off %d+%d size %d\n",
                                regno, min_off, i - min_off, access_size);
                } else {
                        char tn_buf[48];

                        tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                        verbose(env, "invalid read from stack R%d var_off %s+%d size %d\n",
                                regno, tn_buf, i - min_off, access_size);
                }
                return -EACCES;
mark:
                /* reading any byte out of 8-byte 'spill_slot' will cause
                 * the whole slot to be marked as 'read'
                 */
                mark_reg_read(env, &state->stack[spi].spilled_ptr,
                              state->stack[spi].spilled_ptr.parent,
                              REG_LIVE_READ64);
                /* We do not set REG_LIVE_WRITTEN for stack slot, as we can not
                 * be sure that whether stack slot is written to or not. Hence,
                 * we must still conservatively propagate reads upwards even if
                 * helper may write to the entire memory range.
                 */
        }
        return 0;
}

static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
                                   int access_size, enum bpf_access_type access_type,
                                   bool zero_size_allowed,
                                   struct bpf_call_arg_meta *meta)
{
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        u32 *max_access;

        switch (base_type(reg->type)) {
        case PTR_TO_PACKET:
        case PTR_TO_PACKET_META:
                return check_packet_access(env, regno, reg->off, access_size,
                                           zero_size_allowed);
        case PTR_TO_MAP_KEY:
                if (access_type == BPF_WRITE) {
                        verbose(env, "R%d cannot write into %s\n", regno,
                                reg_type_str(env, reg->type));
                        return -EACCES;
                }
                return check_mem_region_access(env, regno, reg->off, access_size,
                                               reg->map_ptr->key_size, false);
        case PTR_TO_MAP_VALUE:
                if (check_map_access_type(env, regno, reg->off, access_size, access_type))
                        return -EACCES;
                return check_map_access(env, regno, reg->off, access_size,
                                        zero_size_allowed, ACCESS_HELPER);
        case PTR_TO_MEM:
                if (type_is_rdonly_mem(reg->type)) {
                        if (access_type == BPF_WRITE) {
                                verbose(env, "R%d cannot write into %s\n", regno,
                                        reg_type_str(env, reg->type));
                                return -EACCES;
                        }
                }
                return check_mem_region_access(env, regno, reg->off,
                                               access_size, reg->mem_size,
                                               zero_size_allowed);
        case PTR_TO_BUF:
                if (type_is_rdonly_mem(reg->type)) {
                        if (access_type == BPF_WRITE) {
                                verbose(env, "R%d cannot write into %s\n", regno,
                                        reg_type_str(env, reg->type));
                                return -EACCES;
                        }

                        max_access = &env->prog->aux->max_rdonly_access;
                } else {
                        max_access = &env->prog->aux->max_rdwr_access;
                }
                return check_buffer_access(env, reg, regno, reg->off,
                                           access_size, zero_size_allowed,
                                           max_access);
        case PTR_TO_STACK:
                return check_stack_range_initialized(
                                env,
                                regno, reg->off, access_size,
                                zero_size_allowed, access_type, meta);
        case PTR_TO_BTF_ID:
                return check_ptr_to_btf_access(env, regs, regno, reg->off,
                                               access_size, BPF_READ, -1);
        case PTR_TO_CTX:
                /* in case the function doesn't know how to access the context,
                 * (because we are in a program of type SYSCALL for example), we
                 * can not statically check its size.
                 * Dynamically check it now.
                 */
                if (!env->ops->convert_ctx_access) {
                        int offset = access_size - 1;

                        /* Allow zero-byte read from PTR_TO_CTX */
                        if (access_size == 0)
                                return zero_size_allowed ? 0 : -EACCES;

                        return check_mem_access(env, env->insn_idx, regno, offset, BPF_B,
                                                access_type, -1, false, false);
                }

                fallthrough;
        default: /* scalar_value or invalid ptr */
                /* Allow zero-byte read from NULL, regardless of pointer type */
                if (zero_size_allowed && access_size == 0 &&
                    register_is_null(reg))
                        return 0;

                verbose(env, "R%d type=%s ", regno,
                        reg_type_str(env, reg->type));
                verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
                return -EACCES;
        }
}

/* verify arguments to helpers or kfuncs consisting of a pointer and an access
 * size.
 *
 * @regno is the register containing the access size. regno-1 is the register
 * containing the pointer.
 */
static int check_mem_size_reg(struct bpf_verifier_env *env,
                              struct bpf_reg_state *reg, u32 regno,
                              enum bpf_access_type access_type,
                              bool zero_size_allowed,
                              struct bpf_call_arg_meta *meta)
{
        int err;

        /* This is used to refine r0 return value bounds for helpers
         * that enforce this value as an upper bound on return values.
         * See do_refine_retval_range() for helpers that can refine
         * the return value. C type of helper is u32 so we pull register
         * bound from umax_value however, if negative verifier errors
         * out. Only upper bounds can be learned because retval is an
         * int type and negative retvals are allowed.
         */
        meta->msize_max_value = reg->umax_value;

        /* The register is SCALAR_VALUE; the access check happens using
         * its boundaries. For unprivileged variable accesses, disable
         * raw mode so that the program is required to initialize all
         * the memory that the helper could just partially fill up.
         */
        if (!tnum_is_const(reg->var_off))
                meta = NULL;

        if (reg->smin_value < 0) {
                verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
                        regno);
                return -EACCES;
        }

        if (reg->umin_value == 0 && !zero_size_allowed) {
                verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n",
                        regno, reg->umin_value, reg->umax_value);
                return -EACCES;
        }

        if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
                verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
                        regno);
                return -EACCES;
        }
        err = check_helper_mem_access(env, regno - 1, reg->umax_value,
                                      access_type, zero_size_allowed, meta);
        if (!err)
                err = mark_chain_precision(env, regno);
        return err;
}

static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
                         u32 regno, u32 mem_size)
{
        bool may_be_null = type_may_be_null(reg->type);
        struct bpf_reg_state saved_reg;
        int err;

        if (register_is_null(reg))
                return 0;

        /* Assuming that the register contains a value check if the memory
         * access is safe. Temporarily save and restore the register's state as
         * the conversion shouldn't be visible to a caller.
         */
        if (may_be_null) {
                saved_reg = *reg;
                mark_ptr_not_null_reg(reg);
        }

        err = check_helper_mem_access(env, regno, mem_size, BPF_READ, true, NULL);
        err = err ?: check_helper_mem_access(env, regno, mem_size, BPF_WRITE, true, NULL);

        if (may_be_null)
                *reg = saved_reg;

        return err;
}

static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
                                    u32 regno)
{
        struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1];
        bool may_be_null = type_may_be_null(mem_reg->type);
        struct bpf_reg_state saved_reg;
        struct bpf_call_arg_meta meta;
        int err;

        WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5);

        memset(&meta, 0, sizeof(meta));

        if (may_be_null) {
                saved_reg = *mem_reg;
                mark_ptr_not_null_reg(mem_reg);
        }

        err = check_mem_size_reg(env, reg, regno, BPF_READ, true, &meta);
        err = err ?: check_mem_size_reg(env, reg, regno, BPF_WRITE, true, &meta);

        if (may_be_null)
                *mem_reg = saved_reg;

        return err;
}

/* Implementation details:
 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL.
 * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL.
 * Two bpf_map_lookups (even with the same key) will have different reg->id.
 * Two separate bpf_obj_new will also have different reg->id.
 * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier
 * clears reg->id after value_or_null->value transition, since the verifier only
 * cares about the range of access to valid map value pointer and doesn't care
 * about actual address of the map element.
 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
 * reg->id > 0 after value_or_null->value transition. By doing so
 * two bpf_map_lookups will be considered two different pointers that
 * point to different bpf_spin_locks. Likewise for pointers to allocated objects
 * returned from bpf_obj_new.
 * The verifier allows taking only one bpf_spin_lock at a time to avoid
 * dead-locks.
 * Since only one bpf_spin_lock is allowed the checks are simpler than
 * reg_is_refcounted() logic. The verifier needs to remember only
 * one spin_lock instead of array of acquired_refs.
 * env->cur_state->active_locks remembers which map value element or allocated
 * object got locked and clears it after bpf_spin_unlock.
 */
static int process_spin_lock(struct bpf_verifier_env *env, int regno,
                             bool is_lock)
{
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        struct bpf_verifier_state *cur = env->cur_state;
        bool is_const = tnum_is_const(reg->var_off);
        u64 val = reg->var_off.value;
        struct bpf_map *map = NULL;
        struct btf *btf = NULL;
        struct btf_record *rec;
        int err;

        if (!is_const) {
                verbose(env,
                        "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
                        regno);
                return -EINVAL;
        }
        if (reg->type == PTR_TO_MAP_VALUE) {
                map = reg->map_ptr;
                if (!map->btf) {
                        verbose(env,
                                "map '%s' has to have BTF in order to use bpf_spin_lock\n",
                                map->name);
                        return -EINVAL;
                }
        } else {
                btf = reg->btf;
        }

        rec = reg_btf_record(reg);
        if (!btf_record_has_field(rec, BPF_SPIN_LOCK)) {
                verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local",
                        map ? map->name : "kptr");
                return -EINVAL;
        }
        if (rec->spin_lock_off != val + reg->off) {
                verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n",
                        val + reg->off, rec->spin_lock_off);
                return -EINVAL;
        }
        if (is_lock) {
                void *ptr;

                if (map)
                        ptr = map;
                else
                        ptr = btf;

                if (cur->active_locks) {
                        verbose(env,
                                "Locking two bpf_spin_locks are not allowed\n");
                        return -EINVAL;
                }
                err = acquire_lock_state(env, env->insn_idx, REF_TYPE_LOCK, reg->id, ptr);
                if (err < 0) {
                        verbose(env, "Failed to acquire lock state\n");
                        return err;
                }
        } else {
                void *ptr;

                if (map)
                        ptr = map;
                else
                        ptr = btf;

                if (!cur->active_locks) {
                        verbose(env, "bpf_spin_unlock without taking a lock\n");
                        return -EINVAL;
                }

                if (release_lock_state(env->cur_state, REF_TYPE_LOCK, reg->id, ptr)) {
                        verbose(env, "bpf_spin_unlock of different lock\n");
                        return -EINVAL;
                }

                invalidate_non_owning_refs(env);
        }
        return 0;
}

static int process_timer_func(struct bpf_verifier_env *env, int regno,
                              struct bpf_call_arg_meta *meta)
{
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        bool is_const = tnum_is_const(reg->var_off);
        struct bpf_map *map = reg->map_ptr;
        u64 val = reg->var_off.value;

        if (!is_const) {
                verbose(env,
                        "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n",
                        regno);
                return -EINVAL;
        }
        if (!map->btf) {
                verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n",
                        map->name);
                return -EINVAL;
        }
        if (!btf_record_has_field(map->record, BPF_TIMER)) {
                verbose(env, "map '%s' has no valid bpf_timer\n", map->name);
                return -EINVAL;
        }
        if (map->record->timer_off != val + reg->off) {
                verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n",
                        val + reg->off, map->record->timer_off);
                return -EINVAL;
        }
        if (meta->map_ptr) {
                verbose(env, "verifier bug. Two map pointers in a timer helper\n");
                return -EFAULT;
        }
        meta->map_uid = reg->map_uid;
        meta->map_ptr = map;
        return 0;
}

static int process_wq_func(struct bpf_verifier_env *env, int regno,
                           struct bpf_kfunc_call_arg_meta *meta)
{
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        struct bpf_map *map = reg->map_ptr;
        u64 val = reg->var_off.value;

        if (map->record->wq_off != val + reg->off) {
                verbose(env, "off %lld doesn't point to 'struct bpf_wq' that is at %d\n",
                        val + reg->off, map->record->wq_off);
                return -EINVAL;
        }
        meta->map.uid = reg->map_uid;
        meta->map.ptr = map;
        return 0;
}

static int process_kptr_func(struct bpf_verifier_env *env, int regno,
                             struct bpf_call_arg_meta *meta)
{
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        struct btf_field *kptr_field;
        struct bpf_map *map_ptr;
        struct btf_record *rec;
        u32 kptr_off;

        if (type_is_ptr_alloc_obj(reg->type)) {
                rec = reg_btf_record(reg);
        } else { /* PTR_TO_MAP_VALUE */
                map_ptr = reg->map_ptr;
                if (!map_ptr->btf) {
                        verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n",
                                map_ptr->name);
                        return -EINVAL;
                }
                rec = map_ptr->record;
                meta->map_ptr = map_ptr;
        }

        if (!tnum_is_const(reg->var_off)) {
                verbose(env,
                        "R%d doesn't have constant offset. kptr has to be at the constant offset\n",
                        regno);
                return -EINVAL;
        }

        if (!btf_record_has_field(rec, BPF_KPTR)) {
                verbose(env, "R%d has no valid kptr\n", regno);
                return -EINVAL;
        }

        kptr_off = reg->off + reg->var_off.value;
        kptr_field = btf_record_find(rec, kptr_off, BPF_KPTR);
        if (!kptr_field) {
                verbose(env, "off=%d doesn't point to kptr\n", kptr_off);
                return -EACCES;
        }
        if (kptr_field->type != BPF_KPTR_REF && kptr_field->type != BPF_KPTR_PERCPU) {
                verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off);
                return -EACCES;
        }
        meta->kptr_field = kptr_field;
        return 0;
}

/* There are two register types representing a bpf_dynptr, one is PTR_TO_STACK
 * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR.
 *
 * In both cases we deal with the first 8 bytes, but need to mark the next 8
 * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of
 * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object.
 *
 * Mutability of bpf_dynptr is at two levels, one is at the level of struct
 * bpf_dynptr itself, i.e. whether the helper is receiving a pointer to struct
 * bpf_dynptr or pointer to const struct bpf_dynptr. In the former case, it can
 * mutate the view of the dynptr and also possibly destroy it. In the latter
 * case, it cannot mutate the bpf_dynptr itself but it can still mutate the
 * memory that dynptr points to.
 *
 * The verifier will keep track both levels of mutation (bpf_dynptr's in
 * reg->type and the memory's in reg->dynptr.type), but there is no support for
 * readonly dynptr view yet, hence only the first case is tracked and checked.
 *
 * This is consistent with how C applies the const modifier to a struct object,
 * where the pointer itself inside bpf_dynptr becomes const but not what it
 * points to.
 *
 * Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument
 * type, and declare it as 'const struct bpf_dynptr *' in their prototype.
 */
static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
                               enum bpf_arg_type arg_type, int clone_ref_obj_id)
{
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        int err;

        if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) {
                verbose(env,
                        "arg#%d expected pointer to stack or const struct bpf_dynptr\n",
                        regno - 1);
                return -EINVAL;
        }

        /* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an
         * ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*):
         */
        if ((arg_type & (MEM_UNINIT | MEM_RDONLY)) == (MEM_UNINIT | MEM_RDONLY)) {
                verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n");
                return -EFAULT;
        }

        /*  MEM_UNINIT - Points to memory that is an appropriate candidate for
         *                 constructing a mutable bpf_dynptr object.
         *
         *                 Currently, this is only possible with PTR_TO_STACK
         *                 pointing to a region of at least 16 bytes which doesn't
         *                 contain an existing bpf_dynptr.
         *
         *  MEM_RDONLY - Points to a initialized bpf_dynptr that will not be
         *                 mutated or destroyed. However, the memory it points to
         *                 may be mutated.
         *
         *  None       - Points to a initialized dynptr that can be mutated and
         *                 destroyed, including mutation of the memory it points
         *                 to.
         */
        if (arg_type & MEM_UNINIT) {
                int i;

                if (!is_dynptr_reg_valid_uninit(env, reg)) {
                        verbose(env, "Dynptr has to be an uninitialized dynptr\n");
                        return -EINVAL;
                }

                /* we write BPF_DW bits (8 bytes) at a time */
                for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
                        err = check_mem_access(env, insn_idx, regno,
                                               i, BPF_DW, BPF_WRITE, -1, false, false);
                        if (err)
                                return err;
                }

                err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id);
        } else /* MEM_RDONLY and None case from above */ {
                /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */
                if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) {
                        verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n");
                        return -EINVAL;
                }

                if (!is_dynptr_reg_valid_init(env, reg)) {
                        verbose(env,
                                "Expected an initialized dynptr as arg #%d\n",
                                regno - 1);
                        return -EINVAL;
                }

                /* Fold modifiers (in this case, MEM_RDONLY) when checking expected type */
                if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) {
                        verbose(env,
                                "Expected a dynptr of type %s as arg #%d\n",
                                dynptr_type_str(arg_to_dynptr_type(arg_type)), regno - 1);
                        return -EINVAL;
                }

                err = mark_dynptr_read(env, reg);
        }
        return err;
}

static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi)
{
        struct bpf_func_state *state = func(env, reg);

        return state->stack[spi].spilled_ptr.ref_obj_id;
}

static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY);
}

static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_ITER_NEW;
}

static bool is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_ITER_NEXT;
}

static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_ITER_DESTROY;
}

static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx,
                              const struct btf_param *arg)
{
        /* btf_check_iter_kfuncs() guarantees that first argument of any iter
         * kfunc is iter state pointer
         */
        if (is_iter_kfunc(meta))
                return arg_idx == 0;

        /* iter passed as an argument to a generic kfunc */
        return btf_param_match_suffix(meta->btf, arg, "__iter");
}

static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx,
                            struct bpf_kfunc_call_arg_meta *meta)
{
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        const struct btf_type *t;
        int spi, err, i, nr_slots, btf_id;

        if (reg->type != PTR_TO_STACK) {
                verbose(env, "arg#%d expected pointer to an iterator on stack\n", regno - 1);
                return -EINVAL;
        }

        /* For iter_{new,next,destroy} functions, btf_check_iter_kfuncs()
         * ensures struct convention, so we wouldn't need to do any BTF
         * validation here. But given iter state can be passed as a parameter
         * to any kfunc, if arg has "__iter" suffix, we need to be a bit more
         * conservative here.
         */
        btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, regno - 1);
        if (btf_id < 0) {
                verbose(env, "expected valid iter pointer as arg #%d\n", regno - 1);
                return -EINVAL;
        }
        t = btf_type_by_id(meta->btf, btf_id);
        nr_slots = t->size / BPF_REG_SIZE;

        if (is_iter_new_kfunc(meta)) {
                /* bpf_iter_<type>_new() expects pointer to uninit iter state */
                if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) {
                        verbose(env, "expected uninitialized iter_%s as arg #%d\n",
                                iter_type_str(meta->btf, btf_id), regno - 1);
                        return -EINVAL;
                }

                for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) {
                        err = check_mem_access(env, insn_idx, regno,
                                               i, BPF_DW, BPF_WRITE, -1, false, false);
                        if (err)
                                return err;
                }

                err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots);
                if (err)
                        return err;
        } else {
                /* iter_next() or iter_destroy(), as well as any kfunc
                 * accepting iter argument, expect initialized iter state
                 */
                err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots);
                switch (err) {
                case 0:
                        break;
                case -EINVAL:
                        verbose(env, "expected an initialized iter_%s as arg #%d\n",
                                iter_type_str(meta->btf, btf_id), regno - 1);
                        return err;
                case -EPROTO:
                        verbose(env, "expected an RCU CS when using %s\n", meta->func_name);
                        return err;
                default:
                        return err;
                }

                spi = iter_get_spi(env, reg, nr_slots);
                if (spi < 0)
                        return spi;

                err = mark_iter_read(env, reg, spi, nr_slots);
                if (err)
                        return err;

                /* remember meta->iter info for process_iter_next_call() */
                meta->iter.spi = spi;
                meta->iter.frameno = reg->frameno;
                meta->ref_obj_id = iter_ref_obj_id(env, reg, spi);

                if (is_iter_destroy_kfunc(meta)) {
                        err = unmark_stack_slots_iter(env, reg, nr_slots);
                        if (err)
                                return err;
                }
        }

        return 0;
}

/* Look for a previous loop entry at insn_idx: nearest parent state
 * stopped at insn_idx with callsites matching those in cur->frame.
 */
static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env,
                                                  struct bpf_verifier_state *cur,
                                                  int insn_idx)
{
        struct bpf_verifier_state_list *sl;
        struct bpf_verifier_state *st;

        /* Explored states are pushed in stack order, most recent states come first */
        sl = *explored_state(env, insn_idx);
        for (; sl; sl = sl->next) {
                /* If st->branches != 0 state is a part of current DFS verification path,
                 * hence cur & st for a loop.
                 */
                st = &sl->state;
                if (st->insn_idx == insn_idx && st->branches && same_callsites(st, cur) &&
                    st->dfs_depth < cur->dfs_depth)
                        return st;
        }

        return NULL;
}

static void reset_idmap_scratch(struct bpf_verifier_env *env);
static bool regs_exact(const struct bpf_reg_state *rold,
                       const struct bpf_reg_state *rcur,
                       struct bpf_idmap *idmap);

static void maybe_widen_reg(struct bpf_verifier_env *env,
                            struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
                            struct bpf_idmap *idmap)
{
        if (rold->type != SCALAR_VALUE)
                return;
        if (rold->type != rcur->type)
                return;
        if (rold->precise || rcur->precise || regs_exact(rold, rcur, idmap))
                return;
        __mark_reg_unknown(env, rcur);
}

static int widen_imprecise_scalars(struct bpf_verifier_env *env,
                                   struct bpf_verifier_state *old,
                                   struct bpf_verifier_state *cur)
{
        struct bpf_func_state *fold, *fcur;
        int i, fr;

        reset_idmap_scratch(env);
        for (fr = old->curframe; fr >= 0; fr--) {
                fold = old->frame[fr];
                fcur = cur->frame[fr];

                for (i = 0; i < MAX_BPF_REG; i++)
                        maybe_widen_reg(env,
                                        &fold->regs[i],
                                        &fcur->regs[i],
                                        &env->idmap_scratch);

                for (i = 0; i < fold->allocated_stack / BPF_REG_SIZE; i++) {
                        if (!is_spilled_reg(&fold->stack[i]) ||
                            !is_spilled_reg(&fcur->stack[i]))
                                continue;

                        maybe_widen_reg(env,
                                        &fold->stack[i].spilled_ptr,
                                        &fcur->stack[i].spilled_ptr,
                                        &env->idmap_scratch);
                }
        }
        return 0;
}

static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_st,
                                                 struct bpf_kfunc_call_arg_meta *meta)
{
        int iter_frameno = meta->iter.frameno;
        int iter_spi = meta->iter.spi;

        return &cur_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr;
}

/* process_iter_next_call() is called when verifier gets to iterator's next
 * "method" (e.g., bpf_iter_num_next() for numbers iterator) call. We'll refer
 * to it as just "iter_next()" in comments below.
 *
 * BPF verifier relies on a crucial contract for any iter_next()
 * implementation: it should *eventually* return NULL, and once that happens
 * it should keep returning NULL. That is, once iterator exhausts elements to
 * iterate, it should never reset or spuriously return new elements.
 *
 * With the assumption of such contract, process_iter_next_call() simulates
 * a fork in the verifier state to validate loop logic correctness and safety
 * without having to simulate infinite amount of iterations.
 *
 * In current state, we first assume that iter_next() returned NULL and
 * iterator state is set to DRAINED (BPF_ITER_STATE_DRAINED). In such
 * conditions we should not form an infinite loop and should eventually reach
 * exit.
 *
 * Besides that, we also fork current state and enqueue it for later
 * verification. In a forked state we keep iterator state as ACTIVE
 * (BPF_ITER_STATE_ACTIVE) and assume non-NULL return from iter_next(). We
 * also bump iteration depth to prevent erroneous infinite loop detection
 * later on (see iter_active_depths_differ() comment for details). In this
 * state we assume that we'll eventually loop back to another iter_next()
 * calls (it could be in exactly same location or in some other instruction,
 * it doesn't matter, we don't make any unnecessary assumptions about this,
 * everything revolves around iterator state in a stack slot, not which
 * instruction is calling iter_next()). When that happens, we either will come
 * to iter_next() with equivalent state and can conclude that next iteration
 * will proceed in exactly the same way as we just verified, so it's safe to
 * assume that loop converges. If not, we'll go on another iteration
 * simulation with a different input state, until all possible starting states
 * are validated or we reach maximum number of instructions limit.
 *
 * This way, we will either exhaustively discover all possible input states
 * that iterator loop can start with and eventually will converge, or we'll
 * effectively regress into bounded loop simulation logic and either reach
 * maximum number of instructions if loop is not provably convergent, or there
 * is some statically known limit on number of iterations (e.g., if there is
 * an explicit `if n > 100 then break;` statement somewhere in the loop).
 *
 * Iteration convergence logic in is_state_visited() relies on exact
 * states comparison, which ignores read and precision marks.
 * This is necessary because read and precision marks are not finalized
 * while in the loop. Exact comparison might preclude convergence for
 * simple programs like below:
 *
 *     i = 0;
 *     while(iter_next(&it))
 *       i++;
 *
 * At each iteration step i++ would produce a new distinct state and
 * eventually instruction processing limit would be reached.
 *
 * To avoid such behavior speculatively forget (widen) range for
 * imprecise scalar registers, if those registers were not precise at the
 * end of the previous iteration and do not match exactly.
 *
 * This is a conservative heuristic that allows to verify wide range of programs,
 * however it precludes verification of programs that conjure an
 * imprecise value on the first loop iteration and use it as precise on a second.
 * For example, the following safe program would fail to verify:
 *
 *     struct bpf_num_iter it;
 *     int arr[10];
 *     int i = 0, a = 0;
 *     bpf_iter_num_new(&it, 0, 10);
 *     while (bpf_iter_num_next(&it)) {
 *       if (a == 0) {
 *         a = 1;
 *         i = 7; // Because i changed verifier would forget
 *                // it's range on second loop entry.
 *       } else {
 *         arr[i] = 42; // This would fail to verify.
 *       }
 *     }
 *     bpf_iter_num_destroy(&it);
 */
static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,
                                  struct bpf_kfunc_call_arg_meta *meta)
{
        struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
        struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr;
        struct bpf_reg_state *cur_iter, *queued_iter;

        BTF_TYPE_EMIT(struct bpf_iter);

        cur_iter = get_iter_from_state(cur_st, meta);

        if (cur_iter->iter.state != BPF_ITER_STATE_ACTIVE &&
            cur_iter->iter.state != BPF_ITER_STATE_DRAINED) {
                verbose(env, "verifier internal error: unexpected iterator state %d (%s)\n",
                        cur_iter->iter.state, iter_state_str(cur_iter->iter.state));
                return -EFAULT;
        }

        if (cur_iter->iter.state == BPF_ITER_STATE_ACTIVE) {
                /* Because iter_next() call is a checkpoint is_state_visitied()
                 * should guarantee parent state with same call sites and insn_idx.
                 */
                if (!cur_st->parent || cur_st->parent->insn_idx != insn_idx ||
                    !same_callsites(cur_st->parent, cur_st)) {
                        verbose(env, "bug: bad parent state for iter next call");
                        return -EFAULT;
                }
                /* Note cur_st->parent in the call below, it is necessary to skip
                 * checkpoint created for cur_st by is_state_visited()
                 * right at this instruction.
                 */
                prev_st = find_prev_entry(env, cur_st->parent, insn_idx);
                /* branch out active iter state */
                queued_st = push_stack(env, insn_idx + 1, insn_idx, false);
                if (!queued_st)
                        return -ENOMEM;

                queued_iter = get_iter_from_state(queued_st, meta);
                queued_iter->iter.state = BPF_ITER_STATE_ACTIVE;
                queued_iter->iter.depth++;
                if (prev_st)
                        widen_imprecise_scalars(env, prev_st, queued_st);

                queued_fr = queued_st->frame[queued_st->curframe];
                mark_ptr_not_null_reg(&queued_fr->regs[BPF_REG_0]);
        }

        /* switch to DRAINED state, but keep the depth unchanged */
        /* mark current iter state as drained and assume returned NULL */
        cur_iter->iter.state = BPF_ITER_STATE_DRAINED;
        __mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]);

        return 0;
}

static bool arg_type_is_mem_size(enum bpf_arg_type type)
{
        return type == ARG_CONST_SIZE ||
               type == ARG_CONST_SIZE_OR_ZERO;
}

static bool arg_type_is_raw_mem(enum bpf_arg_type type)
{
        return base_type(type) == ARG_PTR_TO_MEM &&
               type & MEM_UNINIT;
}

static bool arg_type_is_release(enum bpf_arg_type type)
{
        return type & OBJ_RELEASE;
}

static bool arg_type_is_dynptr(enum bpf_arg_type type)
{
        return base_type(type) == ARG_PTR_TO_DYNPTR;
}

static int resolve_map_arg_type(struct bpf_verifier_env *env,
                                 const struct bpf_call_arg_meta *meta,
                                 enum bpf_arg_type *arg_type)
{
        if (!meta->map_ptr) {
                /* kernel subsystem misconfigured verifier */
                verbose(env, "invalid map_ptr to access map->type\n");
                return -EACCES;
        }

        switch (meta->map_ptr->map_type) {
        case BPF_MAP_TYPE_SOCKMAP:
        case BPF_MAP_TYPE_SOCKHASH:
                if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
                        *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON;
                } else {
                        verbose(env, "invalid arg_type for sockmap/sockhash\n");
                        return -EINVAL;
                }
                break;
        case BPF_MAP_TYPE_BLOOM_FILTER:
                if (meta->func_id == BPF_FUNC_map_peek_elem)
                        *arg_type = ARG_PTR_TO_MAP_VALUE;
                break;
        default:
                break;
        }
        return 0;
}

struct bpf_reg_types {
        const enum bpf_reg_type types[10];
        u32 *btf_id;
};

static const struct bpf_reg_types sock_types = {
        .types = {
                PTR_TO_SOCK_COMMON,
                PTR_TO_SOCKET,
                PTR_TO_TCP_SOCK,
                PTR_TO_XDP_SOCK,
        },
};

#ifdef CONFIG_NET
static const struct bpf_reg_types btf_id_sock_common_types = {
        .types = {
                PTR_TO_SOCK_COMMON,
                PTR_TO_SOCKET,
                PTR_TO_TCP_SOCK,
                PTR_TO_XDP_SOCK,
                PTR_TO_BTF_ID,
                PTR_TO_BTF_ID | PTR_TRUSTED,
        },
        .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
};
#endif

static const struct bpf_reg_types mem_types = {
        .types = {
                PTR_TO_STACK,
                PTR_TO_PACKET,
                PTR_TO_PACKET_META,
                PTR_TO_MAP_KEY,
                PTR_TO_MAP_VALUE,
                PTR_TO_MEM,
                PTR_TO_MEM | MEM_RINGBUF,
                PTR_TO_BUF,
                PTR_TO_BTF_ID | PTR_TRUSTED,
        },
};

static const struct bpf_reg_types spin_lock_types = {
        .types = {
                PTR_TO_MAP_VALUE,
                PTR_TO_BTF_ID | MEM_ALLOC,
        }
};

static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } };
static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } };
static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } };
static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } };
static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
static const struct bpf_reg_types btf_ptr_types = {
        .types = {
                PTR_TO_BTF_ID,
                PTR_TO_BTF_ID | PTR_TRUSTED,
                PTR_TO_BTF_ID | MEM_RCU,
        },
};
static const struct bpf_reg_types percpu_btf_ptr_types = {
        .types = {
                PTR_TO_BTF_ID | MEM_PERCPU,
                PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU,
                PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED,
        }
};
static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };
static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };
static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } };
static const struct bpf_reg_types kptr_xchg_dest_types = {
        .types = {
                PTR_TO_MAP_VALUE,
                PTR_TO_BTF_ID | MEM_ALLOC
        }
};
static const struct bpf_reg_types dynptr_types = {
        .types = {
                PTR_TO_STACK,
                CONST_PTR_TO_DYNPTR,
        }
};

static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
        [ARG_PTR_TO_MAP_KEY]                = &mem_types,
        [ARG_PTR_TO_MAP_VALUE]                = &mem_types,
        [ARG_CONST_SIZE]                = &scalar_types,
        [ARG_CONST_SIZE_OR_ZERO]        = &scalar_types,
        [ARG_CONST_ALLOC_SIZE_OR_ZERO]        = &scalar_types,
        [ARG_CONST_MAP_PTR]                = &const_map_ptr_types,
        [ARG_PTR_TO_CTX]                = &context_types,
        [ARG_PTR_TO_SOCK_COMMON]        = &sock_types,
#ifdef CONFIG_NET
        [ARG_PTR_TO_BTF_ID_SOCK_COMMON]        = &btf_id_sock_common_types,
#endif
        [ARG_PTR_TO_SOCKET]                = &fullsock_types,
        [ARG_PTR_TO_BTF_ID]                = &btf_ptr_types,
        [ARG_PTR_TO_SPIN_LOCK]                = &spin_lock_types,
        [ARG_PTR_TO_MEM]                = &mem_types,
        [ARG_PTR_TO_RINGBUF_MEM]        = &ringbuf_mem_types,
        [ARG_PTR_TO_PERCPU_BTF_ID]        = &percpu_btf_ptr_types,
        [ARG_PTR_TO_FUNC]                = &func_ptr_types,
        [ARG_PTR_TO_STACK]                = &stack_ptr_types,
        [ARG_PTR_TO_CONST_STR]                = &const_str_ptr_types,
        [ARG_PTR_TO_TIMER]                = &timer_types,
        [ARG_KPTR_XCHG_DEST]                = &kptr_xchg_dest_types,
        [ARG_PTR_TO_DYNPTR]                = &dynptr_types,
};

static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
                          enum bpf_arg_type arg_type,
                          const u32 *arg_btf_id,
                          struct bpf_call_arg_meta *meta)
{
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        enum bpf_reg_type expected, type = reg->type;
        const struct bpf_reg_types *compatible;
        int i, j;

        compatible = compatible_reg_types[base_type(arg_type)];
        if (!compatible) {
                verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
                return -EFAULT;
        }

        /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
         * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
         *
         * Same for MAYBE_NULL:
         *
         * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
         * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
         *
         * ARG_PTR_TO_MEM is compatible with PTR_TO_MEM that is tagged with a dynptr type.
         *
         * Therefore we fold these flags depending on the arg_type before comparison.
         */
        if (arg_type & MEM_RDONLY)
                type &= ~MEM_RDONLY;
        if (arg_type & PTR_MAYBE_NULL)
                type &= ~PTR_MAYBE_NULL;
        if (base_type(arg_type) == ARG_PTR_TO_MEM)
                type &= ~DYNPTR_TYPE_FLAG_MASK;

        /* Local kptr types are allowed as the source argument of bpf_kptr_xchg */
        if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && regno == BPF_REG_2) {
                type &= ~MEM_ALLOC;
                type &= ~MEM_PERCPU;
        }

        for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
                expected = compatible->types[i];
                if (expected == NOT_INIT)
                        break;

                if (type == expected)
                        goto found;
        }

        verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
        for (j = 0; j + 1 < i; j++)
                verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
        verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
        return -EACCES;

found:
        if (base_type(reg->type) != PTR_TO_BTF_ID)
                return 0;

        if (compatible == &mem_types) {
                if (!(arg_type & MEM_RDONLY)) {
                        verbose(env,
                                "%s() may write into memory pointed by R%d type=%s\n",
                                func_id_name(meta->func_id),
                                regno, reg_type_str(env, reg->type));
                        return -EACCES;
                }
                return 0;
        }

        switch ((int)reg->type) {
        case PTR_TO_BTF_ID:
        case PTR_TO_BTF_ID | PTR_TRUSTED:
        case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL:
        case PTR_TO_BTF_ID | MEM_RCU:
        case PTR_TO_BTF_ID | PTR_MAYBE_NULL:
        case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU:
        {
                /* For bpf_sk_release, it needs to match against first member
                 * 'struct sock_common', hence make an exception for it. This
                 * allows bpf_sk_release to work for multiple socket types.
                 */
                bool strict_type_match = arg_type_is_release(arg_type) &&
                                         meta->func_id != BPF_FUNC_sk_release;

                if (type_may_be_null(reg->type) &&
                    (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) {
                        verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno);
                        return -EACCES;
                }

                if (!arg_btf_id) {
                        if (!compatible->btf_id) {
                                verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
                                return -EFAULT;
                        }
                        arg_btf_id = compatible->btf_id;
                }

                if (meta->func_id == BPF_FUNC_kptr_xchg) {
                        if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
                                return -EACCES;
                } else {
                        if (arg_btf_id == BPF_PTR_POISON) {
                                verbose(env, "verifier internal error:");
                                verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n",
                                        regno);
                                return -EACCES;
                        }

                        if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
                                                  btf_vmlinux, *arg_btf_id,
                                                  strict_type_match)) {
                                verbose(env, "R%d is of type %s but %s is expected\n",
                                        regno, btf_type_name(reg->btf, reg->btf_id),
                                        btf_type_name(btf_vmlinux, *arg_btf_id));
                                return -EACCES;
                        }
                }
                break;
        }
        case PTR_TO_BTF_ID | MEM_ALLOC:
        case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC:
                if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock &&
                    meta->func_id != BPF_FUNC_kptr_xchg) {
                        verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n");
                        return -EFAULT;
                }
                /* Check if local kptr in src arg matches kptr in dst arg */
                if (meta->func_id == BPF_FUNC_kptr_xchg && regno == BPF_REG_2) {
                        if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
                                return -EACCES;
                }
                break;
        case PTR_TO_BTF_ID | MEM_PERCPU:
        case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU:
        case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED:
                /* Handled by helper specific checks */
                break;
        default:
                verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n");
                return -EFAULT;
        }
        return 0;
}

static struct btf_field *
reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields)
{
        struct btf_field *field;
        struct btf_record *rec;

        rec = reg_btf_record(reg);
        if (!rec)
                return NULL;

        field = btf_record_find(rec, off, fields);
        if (!field)
                return NULL;

        return field;
}

static int check_func_arg_reg_off(struct bpf_verifier_env *env,
                                  const struct bpf_reg_state *reg, int regno,
                                  enum bpf_arg_type arg_type)
{
        u32 type = reg->type;

        /* When referenced register is passed to release function, its fixed
         * offset must be 0.
         *
         * We will check arg_type_is_release reg has ref_obj_id when storing
         * meta->release_regno.
         */
        if (arg_type_is_release(arg_type)) {
                /* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it
                 * may not directly point to the object being released, but to
                 * dynptr pointing to such object, which might be at some offset
                 * on the stack. In that case, we simply to fallback to the
                 * default handling.
                 */
                if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK)
                        return 0;

                /* Doing check_ptr_off_reg check for the offset will catch this
                 * because fixed_off_ok is false, but checking here allows us
                 * to give the user a better error message.
                 */
                if (reg->off) {
                        verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n",
                                regno);
                        return -EINVAL;
                }
                return __check_ptr_off_reg(env, reg, regno, false);
        }

        switch (type) {
        /* Pointer types where both fixed and variable offset is explicitly allowed: */
        case PTR_TO_STACK:
        case PTR_TO_PACKET:
        case PTR_TO_PACKET_META:
        case PTR_TO_MAP_KEY:
        case PTR_TO_MAP_VALUE:
        case PTR_TO_MEM:
        case PTR_TO_MEM | MEM_RDONLY:
        case PTR_TO_MEM | MEM_RINGBUF:
        case PTR_TO_BUF:
        case PTR_TO_BUF | MEM_RDONLY:
        case PTR_TO_ARENA:
        case SCALAR_VALUE:
                return 0;
        /* All the rest must be rejected, except PTR_TO_BTF_ID which allows
         * fixed offset.
         */
        case PTR_TO_BTF_ID:
        case PTR_TO_BTF_ID | MEM_ALLOC:
        case PTR_TO_BTF_ID | PTR_TRUSTED:
        case PTR_TO_BTF_ID | MEM_RCU:
        case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF:
        case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU:
                /* When referenced PTR_TO_BTF_ID is passed to release function,
                 * its fixed offset must be 0. In the other cases, fixed offset
                 * can be non-zero. This was already checked above. So pass
                 * fixed_off_ok as true to allow fixed offset for all other
                 * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we
                 * still need to do checks instead of returning.
                 */
                return __check_ptr_off_reg(env, reg, regno, true);
        default:
                return __check_ptr_off_reg(env, reg, regno, false);
        }
}

static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env,
                                                const struct bpf_func_proto *fn,
                                                struct bpf_reg_state *regs)
{
        struct bpf_reg_state *state = NULL;
        int i;

        for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
                if (arg_type_is_dynptr(fn->arg_type[i])) {
                        if (state) {
                                verbose(env, "verifier internal error: multiple dynptr args\n");
                                return NULL;
                        }
                        state = &regs[BPF_REG_1 + i];
                }

        if (!state)
                verbose(env, "verifier internal error: no dynptr arg found\n");

        return state;
}

static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        struct bpf_func_state *state = func(env, reg);
        int spi;

        if (reg->type == CONST_PTR_TO_DYNPTR)
                return reg->id;
        spi = dynptr_get_spi(env, reg);
        if (spi < 0)
                return spi;
        return state->stack[spi].spilled_ptr.id;
}

static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        struct bpf_func_state *state = func(env, reg);
        int spi;

        if (reg->type == CONST_PTR_TO_DYNPTR)
                return reg->ref_obj_id;
        spi = dynptr_get_spi(env, reg);
        if (spi < 0)
                return spi;
        return state->stack[spi].spilled_ptr.ref_obj_id;
}

static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,
                                            struct bpf_reg_state *reg)
{
        struct bpf_func_state *state = func(env, reg);
        int spi;

        if (reg->type == CONST_PTR_TO_DYNPTR)
                return reg->dynptr.type;

        spi = __get_spi(reg->off);
        if (spi < 0) {
                verbose(env, "verifier internal error: invalid spi when querying dynptr type\n");
                return BPF_DYNPTR_TYPE_INVALID;
        }

        return state->stack[spi].spilled_ptr.dynptr.type;
}

static int check_reg_const_str(struct bpf_verifier_env *env,
                               struct bpf_reg_state *reg, u32 regno)
{
        struct bpf_map *map = reg->map_ptr;
        int err;
        int map_off;
        u64 map_addr;
        char *str_ptr;

        if (reg->type != PTR_TO_MAP_VALUE)
                return -EINVAL;

        if (!bpf_map_is_rdonly(map)) {
                verbose(env, "R%d does not point to a readonly map'\n", regno);
                return -EACCES;
        }

        if (!tnum_is_const(reg->var_off)) {
                verbose(env, "R%d is not a constant address'\n", regno);
                return -EACCES;
        }

        if (!map->ops->map_direct_value_addr) {
                verbose(env, "no direct value access support for this map type\n");
                return -EACCES;
        }

        err = check_map_access(env, regno, reg->off,
                               map->value_size - reg->off, false,
                               ACCESS_HELPER);
        if (err)
                return err;

        map_off = reg->off + reg->var_off.value;
        err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
        if (err) {
                verbose(env, "direct value access on string failed\n");
                return err;
        }

        str_ptr = (char *)(long)(map_addr);
        if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
                verbose(env, "string is not zero-terminated\n");
                return -EINVAL;
        }
        return 0;
}

/* Returns constant key value in `value` if possible, else negative error */
static int get_constant_map_key(struct bpf_verifier_env *env,
                                struct bpf_reg_state *key,
                                u32 key_size,
                                s64 *value)
{
        struct bpf_func_state *state = func(env, key);
        struct bpf_reg_state *reg;
        int slot, spi, off;
        int spill_size = 0;
        int zero_size = 0;
        int stack_off;
        int i, err;
        u8 *stype;

        if (!env->bpf_capable)
                return -EOPNOTSUPP;
        if (key->type != PTR_TO_STACK)
                return -EOPNOTSUPP;
        if (!tnum_is_const(key->var_off))
                return -EOPNOTSUPP;

        stack_off = key->off + key->var_off.value;
        slot = -stack_off - 1;
        spi = slot / BPF_REG_SIZE;
        off = slot % BPF_REG_SIZE;
        stype = state->stack[spi].slot_type;

        /* First handle precisely tracked STACK_ZERO */
        for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--)
                zero_size++;
        if (zero_size >= key_size) {
                *value = 0;
                return 0;
        }

        /* Check that stack contains a scalar spill of expected size */
        if (!is_spilled_scalar_reg(&state->stack[spi]))
                return -EOPNOTSUPP;
        for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--)
                spill_size++;
        if (spill_size != key_size)
                return -EOPNOTSUPP;

        reg = &state->stack[spi].spilled_ptr;
        if (!tnum_is_const(reg->var_off))
                /* Stack value not statically known */
                return -EOPNOTSUPP;

        /* We are relying on a constant value. So mark as precise
         * to prevent pruning on it.
         */
        bt_set_frame_slot(&env->bt, key->frameno, spi);
        err = mark_chain_precision_batch(env);
        if (err < 0)
                return err;

        *value = reg->var_off.value;
        return 0;
}

static bool can_elide_value_nullness(enum bpf_map_type type);

static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
                          struct bpf_call_arg_meta *meta,
                          const struct bpf_func_proto *fn,
                          int insn_idx)
{
        u32 regno = BPF_REG_1 + arg;
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        enum bpf_arg_type arg_type = fn->arg_type[arg];
        enum bpf_reg_type type = reg->type;
        u32 *arg_btf_id = NULL;
        u32 key_size;
        int err = 0;

        if (arg_type == ARG_DONTCARE)
                return 0;

        err = check_reg_arg(env, regno, SRC_OP);
        if (err)
                return err;

        if (arg_type == ARG_ANYTHING) {
                if (is_pointer_value(env, regno)) {
                        verbose(env, "R%d leaks addr into helper function\n",
                                regno);
                        return -EACCES;
                }
                return 0;
        }

        if (type_is_pkt_pointer(type) &&
            !may_access_direct_pkt_data(env, meta, BPF_READ)) {
                verbose(env, "helper access to the packet is not allowed\n");
                return -EACCES;
        }

        if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) {
                err = resolve_map_arg_type(env, meta, &arg_type);
                if (err)
                        return err;
        }

        if (register_is_null(reg) && type_may_be_null(arg_type))
                /* A NULL register has a SCALAR_VALUE type, so skip
                 * type checking.
                 */
                goto skip_type_check;

        /* arg_btf_id and arg_size are in a union. */
        if (base_type(arg_type) == ARG_PTR_TO_BTF_ID ||
            base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
                arg_btf_id = fn->arg_btf_id[arg];

        err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
        if (err)
                return err;

        err = check_func_arg_reg_off(env, reg, regno, arg_type);
        if (err)
                return err;

skip_type_check:
        if (arg_type_is_release(arg_type)) {
                if (arg_type_is_dynptr(arg_type)) {
                        struct bpf_func_state *state = func(env, reg);
                        int spi;

                        /* Only dynptr created on stack can be released, thus
                         * the get_spi and stack state checks for spilled_ptr
                         * should only be done before process_dynptr_func for
                         * PTR_TO_STACK.
                         */
                        if (reg->type == PTR_TO_STACK) {
                                spi = dynptr_get_spi(env, reg);
                                if (spi < 0 || !state->stack[spi].spilled_ptr.ref_obj_id) {
                                        verbose(env, "arg %d is an unacquired reference\n", regno);
                                        return -EINVAL;
                                }
                        } else {
                                verbose(env, "cannot release unowned const bpf_dynptr\n");
                                return -EINVAL;
                        }
                } else if (!reg->ref_obj_id && !register_is_null(reg)) {
                        verbose(env, "R%d must be referenced when passed to release function\n",
                                regno);
                        return -EINVAL;
                }
                if (meta->release_regno) {
                        verbose(env, "verifier internal error: more than one release argument\n");
                        return -EFAULT;
                }
                meta->release_regno = regno;
        }

        if (reg->ref_obj_id && base_type(arg_type) != ARG_KPTR_XCHG_DEST) {
                if (meta->ref_obj_id) {
                        verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
                                regno, reg->ref_obj_id,
                                meta->ref_obj_id);
                        return -EFAULT;
                }
                meta->ref_obj_id = reg->ref_obj_id;
        }

        switch (base_type(arg_type)) {
        case ARG_CONST_MAP_PTR:
                /* bpf_map_xxx(map_ptr) call: remember that map_ptr */
                if (meta->map_ptr) {
                        /* Use map_uid (which is unique id of inner map) to reject:
                         * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
                         * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
                         * if (inner_map1 && inner_map2) {
                         *     timer = bpf_map_lookup_elem(inner_map1);
                         *     if (timer)
                         *         // mismatch would have been allowed
                         *         bpf_timer_init(timer, inner_map2);
                         * }
                         *
                         * Comparing map_ptr is enough to distinguish normal and outer maps.
                         */
                        if (meta->map_ptr != reg->map_ptr ||
                            meta->map_uid != reg->map_uid) {
                                verbose(env,
                                        "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
                                        meta->map_uid, reg->map_uid);
                                return -EINVAL;
                        }
                }
                meta->map_ptr = reg->map_ptr;
                meta->map_uid = reg->map_uid;
                break;
        case ARG_PTR_TO_MAP_KEY:
                /* bpf_map_xxx(..., map_ptr, ..., key) call:
                 * check that [key, key + map->key_size) are within
                 * stack limits and initialized
                 */
                if (!meta->map_ptr) {
                        /* in function declaration map_ptr must come before
                         * map_key, so that it's verified and known before
                         * we have to check map_key here. Otherwise it means
                         * that kernel subsystem misconfigured verifier
                         */
                        verbose(env, "invalid map_ptr to access map->key\n");
                        return -EACCES;
                }
                key_size = meta->map_ptr->key_size;
                err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL);
                if (err)
                        return err;
                if (can_elide_value_nullness(meta->map_ptr->map_type)) {
                        err = get_constant_map_key(env, reg, key_size, &meta->const_map_key);
                        if (err < 0) {
                                meta->const_map_key = -1;
                                if (err == -EOPNOTSUPP)
                                        err = 0;
                                else
                                        return err;
                        }
                }
                break;
        case ARG_PTR_TO_MAP_VALUE:
                if (type_may_be_null(arg_type) && register_is_null(reg))
                        return 0;

                /* bpf_map_xxx(..., map_ptr, ..., value) call:
                 * check [value, value + map->value_size) validity
                 */
                if (!meta->map_ptr) {
                        /* kernel subsystem misconfigured verifier */
                        verbose(env, "invalid map_ptr to access map->value\n");
                        return -EACCES;
                }
                meta->raw_mode = arg_type & MEM_UNINIT;
                err = check_helper_mem_access(env, regno, meta->map_ptr->value_size,
                                              arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
                                              false, meta);
                break;
        case ARG_PTR_TO_PERCPU_BTF_ID:
                if (!reg->btf_id) {
                        verbose(env, "Helper has invalid btf_id in R%d\n", regno);
                        return -EACCES;
                }
                meta->ret_btf = reg->btf;
                meta->ret_btf_id = reg->btf_id;
                break;
        case ARG_PTR_TO_SPIN_LOCK:
                if (in_rbtree_lock_required_cb(env)) {
                        verbose(env, "can't spin_{lock,unlock} in rbtree cb\n");
                        return -EACCES;
                }
                if (meta->func_id == BPF_FUNC_spin_lock) {
                        err = process_spin_lock(env, regno, true);
                        if (err)
                                return err;
                } else if (meta->func_id == BPF_FUNC_spin_unlock) {
                        err = process_spin_lock(env, regno, false);
                        if (err)
                                return err;
                } else {
                        verbose(env, "verifier internal error\n");
                        return -EFAULT;
                }
                break;
        case ARG_PTR_TO_TIMER:
                err = process_timer_func(env, regno, meta);
                if (err)
                        return err;
                break;
        case ARG_PTR_TO_FUNC:
                meta->subprogno = reg->subprogno;
                break;
        case ARG_PTR_TO_MEM:
                /* The access to this pointer is only checked when we hit the
                 * next is_mem_size argument below.
                 */
                meta->raw_mode = arg_type & MEM_UNINIT;
                if (arg_type & MEM_FIXED_SIZE) {
                        err = check_helper_mem_access(env, regno, fn->arg_size[arg],
                                                      arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
                                                      false, meta);
                        if (err)
                                return err;
                        if (arg_type & MEM_ALIGNED)
                                err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true);
                }
                break;
        case ARG_CONST_SIZE:
                err = check_mem_size_reg(env, reg, regno,
                                         fn->arg_type[arg - 1] & MEM_WRITE ?
                                         BPF_WRITE : BPF_READ,
                                         false, meta);
                break;
        case ARG_CONST_SIZE_OR_ZERO:
                err = check_mem_size_reg(env, reg, regno,
                                         fn->arg_type[arg - 1] & MEM_WRITE ?
                                         BPF_WRITE : BPF_READ,
                                         true, meta);
                break;
        case ARG_PTR_TO_DYNPTR:
                err = process_dynptr_func(env, regno, insn_idx, arg_type, 0);
                if (err)
                        return err;
                break;
        case ARG_CONST_ALLOC_SIZE_OR_ZERO:
                if (!tnum_is_const(reg->var_off)) {
                        verbose(env, "R%d is not a known constant'\n",
                                regno);
                        return -EACCES;
                }
                meta->mem_size = reg->var_off.value;
                err = mark_chain_precision(env, regno);
                if (err)
                        return err;
                break;
        case ARG_PTR_TO_CONST_STR:
        {
                err = check_reg_const_str(env, reg, regno);
                if (err)
                        return err;
                break;
        }
        case ARG_KPTR_XCHG_DEST:
                err = process_kptr_func(env, regno, meta);
                if (err)
                        return err;
                break;
        }

        return err;
}

static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
{
        enum bpf_attach_type eatype = env->prog->expected_attach_type;
        enum bpf_prog_type type = resolve_prog_type(env->prog);

        if (func_id != BPF_FUNC_map_update_elem &&
            func_id != BPF_FUNC_map_delete_elem)
                return false;

        /* It's not possible to get access to a locked struct sock in these
         * contexts, so updating is safe.
         */
        switch (type) {
        case BPF_PROG_TYPE_TRACING:
                if (eatype == BPF_TRACE_ITER)
                        return true;
                break;
        case BPF_PROG_TYPE_SOCK_OPS:
                /* map_update allowed only via dedicated helpers with event type checks */
                if (func_id == BPF_FUNC_map_delete_elem)
                        return true;
                break;
        case BPF_PROG_TYPE_SOCKET_FILTER:
        case BPF_PROG_TYPE_SCHED_CLS:
        case BPF_PROG_TYPE_SCHED_ACT:
        case BPF_PROG_TYPE_XDP:
        case BPF_PROG_TYPE_SK_REUSEPORT:
        case BPF_PROG_TYPE_FLOW_DISSECTOR:
        case BPF_PROG_TYPE_SK_LOOKUP:
                return true;
        default:
                break;
        }

        verbose(env, "cannot update sockmap in this context\n");
        return false;
}

static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)
{
        return env->prog->jit_requested &&
               bpf_jit_supports_subprog_tailcalls();
}

static int check_map_func_compatibility(struct bpf_verifier_env *env,
                                        struct bpf_map *map, int func_id)
{
        if (!map)
                return 0;

        /* We need a two way check, first is from map perspective ... */
        switch (map->map_type) {
        case BPF_MAP_TYPE_PROG_ARRAY:
                if (func_id != BPF_FUNC_tail_call)
                        goto error;
                break;
        case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
                if (func_id != BPF_FUNC_perf_event_read &&
                    func_id != BPF_FUNC_perf_event_output &&
                    func_id != BPF_FUNC_skb_output &&
                    func_id != BPF_FUNC_perf_event_read_value &&
                    func_id != BPF_FUNC_xdp_output)
                        goto error;
                break;
        case BPF_MAP_TYPE_RINGBUF:
                if (func_id != BPF_FUNC_ringbuf_output &&
                    func_id != BPF_FUNC_ringbuf_reserve &&
                    func_id != BPF_FUNC_ringbuf_query &&
                    func_id != BPF_FUNC_ringbuf_reserve_dynptr &&
                    func_id != BPF_FUNC_ringbuf_submit_dynptr &&
                    func_id != BPF_FUNC_ringbuf_discard_dynptr)
                        goto error;
                break;
        case BPF_MAP_TYPE_USER_RINGBUF:
                if (func_id != BPF_FUNC_user_ringbuf_drain)
                        goto error;
                break;
        case BPF_MAP_TYPE_STACK_TRACE:
                if (func_id != BPF_FUNC_get_stackid)
                        goto error;
                break;
        case BPF_MAP_TYPE_CGROUP_ARRAY:
                if (func_id != BPF_FUNC_skb_under_cgroup &&
                    func_id != BPF_FUNC_current_task_under_cgroup)
                        goto error;
                break;
        case BPF_MAP_TYPE_CGROUP_STORAGE:
        case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
                if (func_id != BPF_FUNC_get_local_storage)
                        goto error;
                break;
        case BPF_MAP_TYPE_DEVMAP:
        case BPF_MAP_TYPE_DEVMAP_HASH:
                if (func_id != BPF_FUNC_redirect_map &&
                    func_id != BPF_FUNC_map_lookup_elem)
                        goto error;
                break;
        /* Restrict bpf side of cpumap and xskmap, open when use-cases
         * appear.
         */
        case BPF_MAP_TYPE_CPUMAP:
                if (func_id != BPF_FUNC_redirect_map)
                        goto error;
                break;
        case BPF_MAP_TYPE_XSKMAP:
                if (func_id != BPF_FUNC_redirect_map &&
                    func_id != BPF_FUNC_map_lookup_elem)
                        goto error;
                break;
        case BPF_MAP_TYPE_ARRAY_OF_MAPS:
        case BPF_MAP_TYPE_HASH_OF_MAPS:
                if (func_id != BPF_FUNC_map_lookup_elem)
                        goto error;
                break;
        case BPF_MAP_TYPE_SOCKMAP:
                if (func_id != BPF_FUNC_sk_redirect_map &&
                    func_id != BPF_FUNC_sock_map_update &&
                    func_id != BPF_FUNC_msg_redirect_map &&
                    func_id != BPF_FUNC_sk_select_reuseport &&
                    func_id != BPF_FUNC_map_lookup_elem &&
                    !may_update_sockmap(env, func_id))
                        goto error;
                break;
        case BPF_MAP_TYPE_SOCKHASH:
                if (func_id != BPF_FUNC_sk_redirect_hash &&
                    func_id != BPF_FUNC_sock_hash_update &&
                    func_id != BPF_FUNC_msg_redirect_hash &&
                    func_id != BPF_FUNC_sk_select_reuseport &&
                    func_id != BPF_FUNC_map_lookup_elem &&
                    !may_update_sockmap(env, func_id))
                        goto error;
                break;
        case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
                if (func_id != BPF_FUNC_sk_select_reuseport)
                        goto error;
                break;
        case BPF_MAP_TYPE_QUEUE:
        case BPF_MAP_TYPE_STACK:
                if (func_id != BPF_FUNC_map_peek_elem &&
                    func_id != BPF_FUNC_map_pop_elem &&
                    func_id != BPF_FUNC_map_push_elem)
                        goto error;
                break;
        case BPF_MAP_TYPE_SK_STORAGE:
                if (func_id != BPF_FUNC_sk_storage_get &&
                    func_id != BPF_FUNC_sk_storage_delete &&
                    func_id != BPF_FUNC_kptr_xchg)
                        goto error;
                break;
        case BPF_MAP_TYPE_INODE_STORAGE:
                if (func_id != BPF_FUNC_inode_storage_get &&
                    func_id != BPF_FUNC_inode_storage_delete &&
                    func_id != BPF_FUNC_kptr_xchg)
                        goto error;
                break;
        case BPF_MAP_TYPE_TASK_STORAGE:
                if (func_id != BPF_FUNC_task_storage_get &&
                    func_id != BPF_FUNC_task_storage_delete &&
                    func_id != BPF_FUNC_kptr_xchg)
                        goto error;
                break;
        case BPF_MAP_TYPE_CGRP_STORAGE:
                if (func_id != BPF_FUNC_cgrp_storage_get &&
                    func_id != BPF_FUNC_cgrp_storage_delete &&
                    func_id != BPF_FUNC_kptr_xchg)
                        goto error;
                break;
        case BPF_MAP_TYPE_BLOOM_FILTER:
                if (func_id != BPF_FUNC_map_peek_elem &&
                    func_id != BPF_FUNC_map_push_elem)
                        goto error;
                break;
        default:
                break;
        }

        /* ... and second from the function itself. */
        switch (func_id) {
        case BPF_FUNC_tail_call:
                if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
                        goto error;
                if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
                        verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
                        return -EINVAL;
                }
                break;
        case BPF_FUNC_perf_event_read:
        case BPF_FUNC_perf_event_output:
        case BPF_FUNC_perf_event_read_value:
        case BPF_FUNC_skb_output:
        case BPF_FUNC_xdp_output:
                if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
                        goto error;
                break;
        case BPF_FUNC_ringbuf_output:
        case BPF_FUNC_ringbuf_reserve:
        case BPF_FUNC_ringbuf_query:
        case BPF_FUNC_ringbuf_reserve_dynptr:
        case BPF_FUNC_ringbuf_submit_dynptr:
        case BPF_FUNC_ringbuf_discard_dynptr:
                if (map->map_type != BPF_MAP_TYPE_RINGBUF)
                        goto error;
                break;
        case BPF_FUNC_user_ringbuf_drain:
                if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF)
                        goto error;
                break;
        case BPF_FUNC_get_stackid:
                if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
                        goto error;
                break;
        case BPF_FUNC_current_task_under_cgroup:
        case BPF_FUNC_skb_under_cgroup:
                if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
                        goto error;
                break;
        case BPF_FUNC_redirect_map:
                if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
                    map->map_type != BPF_MAP_TYPE_DEVMAP_HASH &&
                    map->map_type != BPF_MAP_TYPE_CPUMAP &&
                    map->map_type != BPF_MAP_TYPE_XSKMAP)
                        goto error;
                break;
        case BPF_FUNC_sk_redirect_map:
        case BPF_FUNC_msg_redirect_map:
        case BPF_FUNC_sock_map_update:
                if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
                        goto error;
                break;
        case BPF_FUNC_sk_redirect_hash:
        case BPF_FUNC_msg_redirect_hash:
        case BPF_FUNC_sock_hash_update:
                if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
                        goto error;
                break;
        case BPF_FUNC_get_local_storage:
                if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
                    map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
                        goto error;
                break;
        case BPF_FUNC_sk_select_reuseport:
                if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&
                    map->map_type != BPF_MAP_TYPE_SOCKMAP &&
                    map->map_type != BPF_MAP_TYPE_SOCKHASH)
                        goto error;
                break;
        case BPF_FUNC_map_pop_elem:
                if (map->map_type != BPF_MAP_TYPE_QUEUE &&
                    map->map_type != BPF_MAP_TYPE_STACK)
                        goto error;
                break;
        case BPF_FUNC_map_peek_elem:
        case BPF_FUNC_map_push_elem:
                if (map->map_type != BPF_MAP_TYPE_QUEUE &&
                    map->map_type != BPF_MAP_TYPE_STACK &&
                    map->map_type != BPF_MAP_TYPE_BLOOM_FILTER)
                        goto error;
                break;
        case BPF_FUNC_map_lookup_percpu_elem:
                if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
                    map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
                    map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH)
                        goto error;
                break;
        case BPF_FUNC_sk_storage_get:
        case BPF_FUNC_sk_storage_delete:
                if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
                        goto error;
                break;
        case BPF_FUNC_inode_storage_get:
        case BPF_FUNC_inode_storage_delete:
                if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
                        goto error;
                break;
        case BPF_FUNC_task_storage_get:
        case BPF_FUNC_task_storage_delete:
                if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
                        goto error;
                break;
        case BPF_FUNC_cgrp_storage_get:
        case BPF_FUNC_cgrp_storage_delete:
                if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE)
                        goto error;
                break;
        default:
                break;
        }

        return 0;
error:
        verbose(env, "cannot pass map_type %d into func %s#%d\n",
                map->map_type, func_id_name(func_id), func_id);
        return -EINVAL;
}

static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
{
        int count = 0;

        if (arg_type_is_raw_mem(fn->arg1_type))
                count++;
        if (arg_type_is_raw_mem(fn->arg2_type))
                count++;
        if (arg_type_is_raw_mem(fn->arg3_type))
                count++;
        if (arg_type_is_raw_mem(fn->arg4_type))
                count++;
        if (arg_type_is_raw_mem(fn->arg5_type))
                count++;

        /* We only support one arg being in raw mode at the moment,
         * which is sufficient for the helper functions we have
         * right now.
         */
        return count <= 1;
}

static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
{
        bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;
        bool has_size = fn->arg_size[arg] != 0;
        bool is_next_size = false;

        if (arg + 1 < ARRAY_SIZE(fn->arg_type))
                is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);

        if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM)
                return is_next_size;

        return has_size == is_next_size || is_next_size == is_fixed;
}

static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
{
        /* bpf_xxx(..., buf, len) call will access 'len'
         * bytes from memory 'buf'. Both arg types need
         * to be paired, so make sure there's no buggy
         * helper function specification.
         */
        if (arg_type_is_mem_size(fn->arg1_type) ||
            check_args_pair_invalid(fn, 0) ||
            check_args_pair_invalid(fn, 1) ||
            check_args_pair_invalid(fn, 2) ||
            check_args_pair_invalid(fn, 3) ||
            check_args_pair_invalid(fn, 4))
                return false;

        return true;
}

static bool check_btf_id_ok(const struct bpf_func_proto *fn)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
                if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID)
                        return !!fn->arg_btf_id[i];
                if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK)
                        return fn->arg_btf_id[i] == BPF_PTR_POISON;
                if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] &&
                    /* arg_btf_id and arg_size are in a union. */
                    (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM ||
                     !(fn->arg_type[i] & MEM_FIXED_SIZE)))
                        return false;
        }

        return true;
}

static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
{
        return check_raw_mode_ok(fn) &&
               check_arg_pair_ok(fn) &&
               check_btf_id_ok(fn) ? 0 : -EINVAL;
}

/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
 * are now invalid, so turn them into unknown SCALAR_VALUE.
 *
 * This also applies to dynptr slices belonging to skb and xdp dynptrs,
 * since these slices point to packet data.
 */
static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
{
        struct bpf_func_state *state;
        struct bpf_reg_state *reg;

        bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
                if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))
                        mark_reg_invalid(env, reg);
        }));
}

enum {
        AT_PKT_END = -1,
        BEYOND_PKT_END = -2,
};

static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open)
{
        struct bpf_func_state *state = vstate->frame[vstate->curframe];
        struct bpf_reg_state *reg = &state->regs[regn];

        if (reg->type != PTR_TO_PACKET)
                /* PTR_TO_PACKET_META is not supported yet */
                return;

        /* The 'reg' is pkt > pkt_end or pkt >= pkt_end.
         * How far beyond pkt_end it goes is unknown.
         * if (!range_open) it's the case of pkt >= pkt_end
         * if (range_open) it's the case of pkt > pkt_end
         * hence this pointer is at least 1 byte bigger than pkt_end
         */
        if (range_open)
                reg->range = BEYOND_PKT_END;
        else
                reg->range = AT_PKT_END;
}

static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id)
{
        int i;

        for (i = 0; i < state->acquired_refs; i++) {
                if (state->refs[i].type != REF_TYPE_PTR)
                        continue;
                if (state->refs[i].id == ref_obj_id) {
                        release_reference_state(state, i);
                        return 0;
                }
        }
        return -EINVAL;
}

/* The pointer with the specified id has released its reference to kernel
 * resources. Identify all copies of the same pointer and clear the reference.
 *
 * This is the release function corresponding to acquire_reference(). Idempotent.
 */
static int release_reference(struct bpf_verifier_env *env, int ref_obj_id)
{
        struct bpf_verifier_state *vstate = env->cur_state;
        struct bpf_func_state *state;
        struct bpf_reg_state *reg;
        int err;

        err = release_reference_nomark(vstate, ref_obj_id);
        if (err)
                return err;

        bpf_for_each_reg_in_vstate(vstate, state, reg, ({
                if (reg->ref_obj_id == ref_obj_id)
                        mark_reg_invalid(env, reg);
        }));

        return 0;
}

static void invalidate_non_owning_refs(struct bpf_verifier_env *env)
{
        struct bpf_func_state *unused;
        struct bpf_reg_state *reg;

        bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
                if (type_is_non_owning_ref(reg->type))
                        mark_reg_invalid(env, reg);
        }));
}

static void clear_caller_saved_regs(struct bpf_verifier_env *env,
                                    struct bpf_reg_state *regs)
{
        int i;

        /* after the call registers r0 - r5 were scratched */
        for (i = 0; i < CALLER_SAVED_REGS; i++) {
                mark_reg_not_init(env, regs, caller_saved[i]);
                __check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK);
        }
}

typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
                                   struct bpf_func_state *caller,
                                   struct bpf_func_state *callee,
                                   int insn_idx);

static int set_callee_state(struct bpf_verifier_env *env,
                            struct bpf_func_state *caller,
                            struct bpf_func_state *callee, int insn_idx);

static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,
                            set_callee_state_fn set_callee_state_cb,
                            struct bpf_verifier_state *state)
{
        struct bpf_func_state *caller, *callee;
        int err;

        if (state->curframe + 1 >= MAX_CALL_FRAMES) {
                verbose(env, "the call stack of %d frames is too deep\n",
                        state->curframe + 2);
                return -E2BIG;
        }

        if (state->frame[state->curframe + 1]) {
                verbose(env, "verifier bug. Frame %d already allocated\n",
                        state->curframe + 1);
                return -EFAULT;
        }

        caller = state->frame[state->curframe];
        callee = kzalloc(sizeof(*callee), GFP_KERNEL);
        if (!callee)
                return -ENOMEM;
        state->frame[state->curframe + 1] = callee;

        /* callee cannot access r0, r6 - r9 for reading and has to write
         * into its own stack before reading from it.
         * callee can read/write into caller's stack
         */
        init_func_state(env, callee,
                        /* remember the callsite, it will be used by bpf_exit */
                        callsite,
                        state->curframe + 1 /* frameno within this callchain */,
                        subprog /* subprog number within this prog */);
        err = set_callee_state_cb(env, caller, callee, callsite);
        if (err)
                goto err_out;

        /* only increment it after check_reg_arg() finished */
        state->curframe++;

        return 0;

err_out:
        free_func_state(callee);
        state->frame[state->curframe + 1] = NULL;
        return err;
}

static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
                                    const struct btf *btf,
                                    struct bpf_reg_state *regs)
{
        struct bpf_subprog_info *sub = subprog_info(env, subprog);
        struct bpf_verifier_log *log = &env->log;
        u32 i;
        int ret;

        ret = btf_prepare_func_args(env, subprog);
        if (ret)
                return ret;

        /* check that BTF function arguments match actual types that the
         * verifier sees.
         */
        for (i = 0; i < sub->arg_cnt; i++) {
                u32 regno = i + 1;
                struct bpf_reg_state *reg = &regs[regno];
                struct bpf_subprog_arg_info *arg = &sub->args[i];

                if (arg->arg_type == ARG_ANYTHING) {
                        if (reg->type != SCALAR_VALUE) {
                                bpf_log(log, "R%d is not a scalar\n", regno);
                                return -EINVAL;
                        }
                } else if (arg->arg_type == ARG_PTR_TO_CTX) {
                        ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);
                        if (ret < 0)
                                return ret;
                        /* If function expects ctx type in BTF check that caller
                         * is passing PTR_TO_CTX.
                         */
                        if (reg->type != PTR_TO_CTX) {
                                bpf_log(log, "arg#%d expects pointer to ctx\n", i);
                                return -EINVAL;
                        }
                } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
                        ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);
                        if (ret < 0)
                                return ret;
                        if (check_mem_reg(env, reg, regno, arg->mem_size))
                                return -EINVAL;
                        if (!(arg->arg_type & PTR_MAYBE_NULL) && (reg->type & PTR_MAYBE_NULL)) {
                                bpf_log(log, "arg#%d is expected to be non-NULL\n", i);
                                return -EINVAL;
                        }
                } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
                        /*
                         * Can pass any value and the kernel won't crash, but
                         * only PTR_TO_ARENA or SCALAR make sense. Everything
                         * else is a bug in the bpf program. Point it out to
                         * the user at the verification time instead of
                         * run-time debug nightmare.
                         */
                        if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) {
                                bpf_log(log, "R%d is not a pointer to arena or scalar.\n", regno);
                                return -EINVAL;
                        }
                } else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) {
                        ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR);
                        if (ret)
                                return ret;

                        ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0);
                        if (ret)
                                return ret;
                } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
                        struct bpf_call_arg_meta meta;
                        int err;

                        if (register_is_null(reg) && type_may_be_null(arg->arg_type))
                                continue;

                        memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
                        err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);
                        err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);
                        if (err)
                                return err;
                } else {
                        bpf_log(log, "verifier bug: unrecognized arg#%d type %d\n",
                                i, arg->arg_type);
                        return -EFAULT;
                }
        }

        return 0;
}

/* Compare BTF of a function call with given bpf_reg_state.
 * Returns:
 * EFAULT - there is a verifier bug. Abort verification.
 * EINVAL - there is a type mismatch or BTF is not available.
 * 0 - BTF matches with what bpf_reg_state expects.
 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
 */
static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
                                  struct bpf_reg_state *regs)
{
        struct bpf_prog *prog = env->prog;
        struct btf *btf = prog->aux->btf;
        u32 btf_id;
        int err;

        if (!prog->aux->func_info)
                return -EINVAL;

        btf_id = prog->aux->func_info[subprog].type_id;
        if (!btf_id)
                return -EFAULT;

        if (prog->aux->func_info_aux[subprog].unreliable)
                return -EINVAL;

        err = btf_check_func_arg_match(env, subprog, btf, regs);
        /* Compiler optimizations can remove arguments from static functions
         * or mismatched type can be passed into a global function.
         * In such cases mark the function as unreliable from BTF point of view.
         */
        if (err)
                prog->aux->func_info_aux[subprog].unreliable = true;
        return err;
}

static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                              int insn_idx, int subprog,
                              set_callee_state_fn set_callee_state_cb)
{
        struct bpf_verifier_state *state = env->cur_state, *callback_state;
        struct bpf_func_state *caller, *callee;
        int err;

        caller = state->frame[state->curframe];
        err = btf_check_subprog_call(env, subprog, caller->regs);
        if (err == -EFAULT)
                return err;

        /* set_callee_state is used for direct subprog calls, but we are
         * interested in validating only BPF helpers that can call subprogs as
         * callbacks
         */
        env->subprog_info[subprog].is_cb = true;
        if (bpf_pseudo_kfunc_call(insn) &&
            !is_callback_calling_kfunc(insn->imm)) {
                verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n",
                        func_id_name(insn->imm), insn->imm);
                return -EFAULT;
        } else if (!bpf_pseudo_kfunc_call(insn) &&
                   !is_callback_calling_function(insn->imm)) { /* helper */
                verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n",
                        func_id_name(insn->imm), insn->imm);
                return -EFAULT;
        }

        if (is_async_callback_calling_insn(insn)) {
                struct bpf_verifier_state *async_cb;

                /* there is no real recursion here. timer and workqueue callbacks are async */
                env->subprog_info[subprog].is_async_cb = true;
                async_cb = push_async_cb(env, env->subprog_info[subprog].start,
                                         insn_idx, subprog,
                                         is_bpf_wq_set_callback_impl_kfunc(insn->imm));
                if (!async_cb)
                        return -EFAULT;
                callee = async_cb->frame[0];
                callee->async_entry_cnt = caller->async_entry_cnt + 1;

                /* Convert bpf_timer_set_callback() args into timer callback args */
                err = set_callee_state_cb(env, caller, callee, insn_idx);
                if (err)
                        return err;

                return 0;
        }

        /* for callback functions enqueue entry to callback and
         * proceed with next instruction within current frame.
         */
        callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);
        if (!callback_state)
                return -ENOMEM;

        err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,
                               callback_state);
        if (err)
                return err;

        callback_state->callback_unroll_depth++;
        callback_state->frame[callback_state->curframe - 1]->callback_depth++;
        caller->callback_depth = 0;
        return 0;
}

static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                           int *insn_idx)
{
        struct bpf_verifier_state *state = env->cur_state;
        struct bpf_func_state *caller;
        int err, subprog, target_insn;

        target_insn = *insn_idx + insn->imm + 1;
        subprog = find_subprog(env, target_insn);
        if (subprog < 0) {
                verbose(env, "verifier bug. No program starts at insn %d\n", target_insn);
                return -EFAULT;
        }

        caller = state->frame[state->curframe];
        err = btf_check_subprog_call(env, subprog, caller->regs);
        if (err == -EFAULT)
                return err;
        if (subprog_is_global(env, subprog)) {
                const char *sub_name = subprog_name(env, subprog);

                /* Only global subprogs cannot be called with a lock held. */
                if (env->cur_state->active_locks) {
                        verbose(env, "global function calls are not allowed while holding a lock,\n"
                                     "use static function instead\n");
                        return -EINVAL;
                }

                /* Only global subprogs cannot be called with preemption disabled. */
                if (env->cur_state->active_preempt_locks) {
                        verbose(env, "global function calls are not allowed with preemption disabled,\n"
                                     "use static function instead\n");
                        return -EINVAL;
                }

                if (env->cur_state->active_irq_id) {
                        verbose(env, "global function calls are not allowed with IRQs disabled,\n"
                                     "use static function instead\n");
                        return -EINVAL;
                }

                if (err) {
                        verbose(env, "Caller passes invalid args into func#%d ('%s')\n",
                                subprog, sub_name);
                        return err;
                }

                verbose(env, "Func#%d ('%s') is global and assumed valid.\n",
                        subprog, sub_name);
                if (env->subprog_info[subprog].changes_pkt_data)
                        clear_all_pkt_pointers(env);
                /* mark global subprog for verifying after main prog */
                subprog_aux(env, subprog)->called = true;
                clear_caller_saved_regs(env, caller->regs);

                /* All global functions return a 64-bit SCALAR_VALUE */
                mark_reg_unknown(env, caller->regs, BPF_REG_0);
                caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;

                /* continue with next insn after call */
                return 0;
        }

        /* for regular function entry setup new frame and continue
         * from that frame.
         */
        err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);
        if (err)
                return err;

        clear_caller_saved_regs(env, caller->regs);

        /* and go analyze first insn of the callee */
        *insn_idx = env->subprog_info[subprog].start - 1;

        if (env->log.level & BPF_LOG_LEVEL) {
                verbose(env, "caller:\n");
                print_verifier_state(env, state, caller->frameno, true);
                verbose(env, "callee:\n");
                print_verifier_state(env, state, state->curframe, true);
        }

        return 0;
}

int map_set_for_each_callback_args(struct bpf_verifier_env *env,
                                   struct bpf_func_state *caller,
                                   struct bpf_func_state *callee)
{
        /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn,
         *      void *callback_ctx, u64 flags);
         * callback_fn(struct bpf_map *map, void *key, void *value,
         *      void *callback_ctx);
         */
        callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];

        callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
        __mark_reg_known_zero(&callee->regs[BPF_REG_2]);
        callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;

        callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
        __mark_reg_known_zero(&callee->regs[BPF_REG_3]);
        callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;

        /* pointer to stack or null */
        callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];

        /* unused */
        __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
        return 0;
}

static int set_callee_state(struct bpf_verifier_env *env,
                            struct bpf_func_state *caller,
                            struct bpf_func_state *callee, int insn_idx)
{
        int i;

        /* copy r1 - r5 args that callee can access.  The copy includes parent
         * pointers, which connects us up to the liveness chain
         */
        for (i = BPF_REG_1; i <= BPF_REG_5; i++)
                callee->regs[i] = caller->regs[i];
        return 0;
}

static int set_map_elem_callback_state(struct bpf_verifier_env *env,
                                       struct bpf_func_state *caller,
                                       struct bpf_func_state *callee,
                                       int insn_idx)
{
        struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];
        struct bpf_map *map;
        int err;

        /* valid map_ptr and poison value does not matter */
        map = insn_aux->map_ptr_state.map_ptr;
        if (!map->ops->map_set_for_each_callback_args ||
            !map->ops->map_for_each_callback) {
                verbose(env, "callback function not allowed for map\n");
                return -ENOTSUPP;
        }

        err = map->ops->map_set_for_each_callback_args(env, caller, callee);
        if (err)
                return err;

        callee->in_callback_fn = true;
        callee->callback_ret_range = retval_range(0, 1);
        return 0;
}

static int set_loop_callback_state(struct bpf_verifier_env *env,
                                   struct bpf_func_state *caller,
                                   struct bpf_func_state *callee,
                                   int insn_idx)
{
        /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx,
         *            u64 flags);
         * callback_fn(u64 index, void *callback_ctx);
         */
        callee->regs[BPF_REG_1].type = SCALAR_VALUE;
        callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];

        /* unused */
        __mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
        __mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
        __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

        callee->in_callback_fn = true;
        callee->callback_ret_range = retval_range(0, 1);
        return 0;
}

static int set_timer_callback_state(struct bpf_verifier_env *env,
                                    struct bpf_func_state *caller,
                                    struct bpf_func_state *callee,
                                    int insn_idx)
{
        struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;

        /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn);
         * callback_fn(struct bpf_map *map, void *key, void *value);
         */
        callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
        __mark_reg_known_zero(&callee->regs[BPF_REG_1]);
        callee->regs[BPF_REG_1].map_ptr = map_ptr;

        callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
        __mark_reg_known_zero(&callee->regs[BPF_REG_2]);
        callee->regs[BPF_REG_2].map_ptr = map_ptr;

        callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
        __mark_reg_known_zero(&callee->regs[BPF_REG_3]);
        callee->regs[BPF_REG_3].map_ptr = map_ptr;

        /* unused */
        __mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
        __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
        callee->in_async_callback_fn = true;
        callee->callback_ret_range = retval_range(0, 1);
        return 0;
}

static int set_find_vma_callback_state(struct bpf_verifier_env *env,
                                       struct bpf_func_state *caller,
                                       struct bpf_func_state *callee,
                                       int insn_idx)
{
        /* bpf_find_vma(struct task_struct *task, u64 addr,
         *               void *callback_fn, void *callback_ctx, u64 flags)
         * (callback_fn)(struct task_struct *task,
         *               struct vm_area_struct *vma, void *callback_ctx);
         */
        callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];

        callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID;
        __mark_reg_known_zero(&callee->regs[BPF_REG_2]);
        callee->regs[BPF_REG_2].btf =  btf_vmlinux;
        callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA];

        /* pointer to stack or null */
        callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4];

        /* unused */
        __mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
        __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
        callee->in_callback_fn = true;
        callee->callback_ret_range = retval_range(0, 1);
        return 0;
}

static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,
                                           struct bpf_func_state *caller,
                                           struct bpf_func_state *callee,
                                           int insn_idx)
{
        /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void
         *                          callback_ctx, u64 flags);
         * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx);
         */
        __mark_reg_not_init(env, &callee->regs[BPF_REG_0]);
        mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL);
        callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];

        /* unused */
        __mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
        __mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
        __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);

        callee->in_callback_fn = true;
        callee->callback_ret_range = retval_range(0, 1);
        return 0;
}

static int set_rbtree_add_callback_state(struct bpf_verifier_env *env,
                                         struct bpf_func_state *caller,
                                         struct bpf_func_state *callee,
                                         int insn_idx)
{
        /* void bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
         *                     bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b));
         *
         * 'struct bpf_rb_node *node' arg to bpf_rbtree_add_impl is the same PTR_TO_BTF_ID w/ offset
         * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd
         * by this point, so look at 'root'
         */
        struct btf_field *field;

        field = reg_find_field_offset(&caller->regs[BPF_REG_1], caller->regs[BPF_REG_1].off,
                                      BPF_RB_ROOT);
        if (!field || !field->graph_root.value_btf_id)
                return -EFAULT;

        mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root);
        ref_set_non_owning(env, &callee->regs[BPF_REG_1]);
        mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root);
        ref_set_non_owning(env, &callee->regs[BPF_REG_2]);

        __mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
        __mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
        __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
        callee->in_callback_fn = true;
        callee->callback_ret_range = retval_range(0, 1);
        return 0;
}

static bool is_rbtree_lock_required_kfunc(u32 btf_id);

/* Are we currently verifying the callback for a rbtree helper that must
 * be called with lock held? If so, no need to complain about unreleased
 * lock
 */
static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
{
        struct bpf_verifier_state *state = env->cur_state;
        struct bpf_insn *insn = env->prog->insnsi;
        struct bpf_func_state *callee;
        int kfunc_btf_id;

        if (!state->curframe)
                return false;

        callee = state->frame[state->curframe];

        if (!callee->in_callback_fn)
                return false;

        kfunc_btf_id = insn[callee->callsite].imm;
        return is_rbtree_lock_required_kfunc(kfunc_btf_id);
}

static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg,
                                bool return_32bit)
{
        if (return_32bit)
                return range.minval <= reg->s32_min_value && reg->s32_max_value <= range.maxval;
        else
                return range.minval <= reg->smin_value && reg->smax_value <= range.maxval;
}

static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
{
        struct bpf_verifier_state *state = env->cur_state, *prev_st;
        struct bpf_func_state *caller, *callee;
        struct bpf_reg_state *r0;
        bool in_callback_fn;
        int err;

        callee = state->frame[state->curframe];
        r0 = &callee->regs[BPF_REG_0];
        if (r0->type == PTR_TO_STACK) {
                /* technically it's ok to return caller's stack pointer
                 * (or caller's caller's pointer) back to the caller,
                 * since these pointers are valid. Only current stack
                 * pointer will be invalid as soon as function exits,
                 * but let's be conservative
                 */
                verbose(env, "cannot return stack pointer to the caller\n");
                return -EINVAL;
        }

        caller = state->frame[state->curframe - 1];
        if (callee->in_callback_fn) {
                if (r0->type != SCALAR_VALUE) {
                        verbose(env, "R0 not a scalar value\n");
                        return -EACCES;
                }

                /* we are going to rely on register's precise value */
                err = mark_reg_read(env, r0, r0->parent, REG_LIVE_READ64);
                err = err ?: mark_chain_precision(env, BPF_REG_0);
                if (err)
                        return err;

                /* enforce R0 return value range, and bpf_callback_t returns 64bit */
                if (!retval_range_within(callee->callback_ret_range, r0, false)) {
                        verbose_invalid_scalar(env, r0, callee->callback_ret_range,
                                               "At callback return", "R0");
                        return -EINVAL;
                }
                if (!calls_callback(env, callee->callsite)) {
                        verbose(env, "BUG: in callback at %d, callsite %d !calls_callback\n",
                                *insn_idx, callee->callsite);
                        return -EFAULT;
                }
        } else {
                /* return to the caller whatever r0 had in the callee */
                caller->regs[BPF_REG_0] = *r0;
        }

        /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
         * there function call logic would reschedule callback visit. If iteration
         * converges is_state_visited() would prune that visit eventually.
         */
        in_callback_fn = callee->in_callback_fn;
        if (in_callback_fn)
                *insn_idx = callee->callsite;
        else
                *insn_idx = callee->callsite + 1;

        if (env->log.level & BPF_LOG_LEVEL) {
                verbose(env, "returning from callee:\n");
                print_verifier_state(env, state, callee->frameno, true);
                verbose(env, "to caller at %d:\n", *insn_idx);
                print_verifier_state(env, state, caller->frameno, true);
        }
        /* clear everything in the callee. In case of exceptional exits using
         * bpf_throw, this will be done by copy_verifier_state for extra frames. */
        free_func_state(callee);
        state->frame[state->curframe--] = NULL;

        /* for callbacks widen imprecise scalars to make programs like below verify:
         *
         *   struct ctx { int i; }
         *   void cb(int idx, struct ctx *ctx) { ctx->i++; ... }
         *   ...
         *   struct ctx = { .i = 0; }
         *   bpf_loop(100, cb, &ctx, 0);
         *
         * This is similar to what is done in process_iter_next_call() for open
         * coded iterators.
         */
        prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;
        if (prev_st) {
                err = widen_imprecise_scalars(env, prev_st, state);
                if (err)
                        return err;
        }
        return 0;
}

static int do_refine_retval_range(struct bpf_verifier_env *env,
                                  struct bpf_reg_state *regs, int ret_type,
                                  int func_id,
                                  struct bpf_call_arg_meta *meta)
{
        struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];

        if (ret_type != RET_INTEGER)
                return 0;

        switch (func_id) {
        case BPF_FUNC_get_stack:
        case BPF_FUNC_get_task_stack:
        case BPF_FUNC_probe_read_str:
        case BPF_FUNC_probe_read_kernel_str:
        case BPF_FUNC_probe_read_user_str:
                ret_reg->smax_value = meta->msize_max_value;
                ret_reg->s32_max_value = meta->msize_max_value;
                ret_reg->smin_value = -MAX_ERRNO;
                ret_reg->s32_min_value = -MAX_ERRNO;
                reg_bounds_sync(ret_reg);
                break;
        case BPF_FUNC_get_smp_processor_id:
                ret_reg->umax_value = nr_cpu_ids - 1;
                ret_reg->u32_max_value = nr_cpu_ids - 1;
                ret_reg->smax_value = nr_cpu_ids - 1;
                ret_reg->s32_max_value = nr_cpu_ids - 1;
                ret_reg->umin_value = 0;
                ret_reg->u32_min_value = 0;
                ret_reg->smin_value = 0;
                ret_reg->s32_min_value = 0;
                reg_bounds_sync(ret_reg);
                break;
        }

        return reg_bounds_sanity_check(env, ret_reg, "retval");
}

static int
record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
                int func_id, int insn_idx)
{
        struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
        struct bpf_map *map = meta->map_ptr;

        if (func_id != BPF_FUNC_tail_call &&
            func_id != BPF_FUNC_map_lookup_elem &&
            func_id != BPF_FUNC_map_update_elem &&
            func_id != BPF_FUNC_map_delete_elem &&
            func_id != BPF_FUNC_map_push_elem &&
            func_id != BPF_FUNC_map_pop_elem &&
            func_id != BPF_FUNC_map_peek_elem &&
            func_id != BPF_FUNC_for_each_map_elem &&
            func_id != BPF_FUNC_redirect_map &&
            func_id != BPF_FUNC_map_lookup_percpu_elem)
                return 0;

        if (map == NULL) {
                verbose(env, "kernel subsystem misconfigured verifier\n");
                return -EINVAL;
        }

        /* In case of read-only, some additional restrictions
         * need to be applied in order to prevent altering the
         * state of the map from program side.
         */
        if ((map->map_flags & BPF_F_RDONLY_PROG) &&
            (func_id == BPF_FUNC_map_delete_elem ||
             func_id == BPF_FUNC_map_update_elem ||
             func_id == BPF_FUNC_map_push_elem ||
             func_id == BPF_FUNC_map_pop_elem)) {
                verbose(env, "write into map forbidden\n");
                return -EACCES;
        }

        if (!aux->map_ptr_state.map_ptr)
                bpf_map_ptr_store(aux, meta->map_ptr,
                                  !meta->map_ptr->bypass_spec_v1, false);
        else if (aux->map_ptr_state.map_ptr != meta->map_ptr)
                bpf_map_ptr_store(aux, meta->map_ptr,
                                  !meta->map_ptr->bypass_spec_v1, true);
        return 0;
}

static int
record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
                int func_id, int insn_idx)
{
        struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
        struct bpf_reg_state *regs = cur_regs(env), *reg;
        struct bpf_map *map = meta->map_ptr;
        u64 val, max;
        int err;

        if (func_id != BPF_FUNC_tail_call)
                return 0;
        if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) {
                verbose(env, "kernel subsystem misconfigured verifier\n");
                return -EINVAL;
        }

        reg = &regs[BPF_REG_3];
        val = reg->var_off.value;
        max = map->max_entries;

        if (!(is_reg_const(reg, false) && val < max)) {
                bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
                return 0;
        }

        err = mark_chain_precision(env, BPF_REG_3);
        if (err)
                return err;
        if (bpf_map_key_unseen(aux))
                bpf_map_key_store(aux, val);
        else if (!bpf_map_key_poisoned(aux) &&
                  bpf_map_key_immediate(aux) != val)
                bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
        return 0;
}

static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)
{
        struct bpf_verifier_state *state = env->cur_state;
        bool refs_lingering = false;
        int i;

        if (!exception_exit && cur_func(env)->frameno)
                return 0;

        for (i = 0; i < state->acquired_refs; i++) {
                if (state->refs[i].type != REF_TYPE_PTR)
                        continue;
                verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
                        state->refs[i].id, state->refs[i].insn_idx);
                refs_lingering = true;
        }
        return refs_lingering ? -EINVAL : 0;
}

static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, const char *prefix)
{
        int err;

        if (check_lock && env->cur_state->active_locks) {
                verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix);
                return -EINVAL;
        }

        err = check_reference_leak(env, exception_exit);
        if (err) {
                verbose(env, "%s would lead to reference leak\n", prefix);
                return err;
        }

        if (check_lock && env->cur_state->active_irq_id) {
                verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix);
                return -EINVAL;
        }

        if (check_lock && env->cur_state->active_rcu_lock) {
                verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);
                return -EINVAL;
        }

        if (check_lock && env->cur_state->active_preempt_locks) {
                verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix);
                return -EINVAL;
        }

        return 0;
}

static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
                                   struct bpf_reg_state *regs)
{
        struct bpf_reg_state *fmt_reg = &regs[BPF_REG_3];
        struct bpf_reg_state *data_len_reg = &regs[BPF_REG_5];
        struct bpf_map *fmt_map = fmt_reg->map_ptr;
        struct bpf_bprintf_data data = {};
        int err, fmt_map_off, num_args;
        u64 fmt_addr;
        char *fmt;

        /* data must be an array of u64 */
        if (data_len_reg->var_off.value % 8)
                return -EINVAL;
        num_args = data_len_reg->var_off.value / 8;

        /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const
         * and map_direct_value_addr is set.
         */
        fmt_map_off = fmt_reg->off + fmt_reg->var_off.value;
        err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
                                                  fmt_map_off);
        if (err) {
                verbose(env, "verifier bug\n");
                return -EFAULT;
        }
        fmt = (char *)(long)fmt_addr + fmt_map_off;

        /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we
         * can focus on validating the format specifiers.
         */
        err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data);
        if (err < 0)
                verbose(env, "Invalid format string\n");

        return err;
}

static int check_get_func_ip(struct bpf_verifier_env *env)
{
        enum bpf_prog_type type = resolve_prog_type(env->prog);
        int func_id = BPF_FUNC_get_func_ip;

        if (type == BPF_PROG_TYPE_TRACING) {
                if (!bpf_prog_has_trampoline(env->prog)) {
                        verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",
                                func_id_name(func_id), func_id);
                        return -ENOTSUPP;
                }
                return 0;
        } else if (type == BPF_PROG_TYPE_KPROBE) {
                return 0;
        }

        verbose(env, "func %s#%d not supported for program type %d\n",
                func_id_name(func_id), func_id, type);
        return -ENOTSUPP;
}

static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
{
        return &env->insn_aux_data[env->insn_idx];
}

static bool loop_flag_is_zero(struct bpf_verifier_env *env)
{
        struct bpf_reg_state *regs = cur_regs(env);
        struct bpf_reg_state *reg = &regs[BPF_REG_4];
        bool reg_is_null = register_is_null(reg);

        if (reg_is_null)
                mark_chain_precision(env, BPF_REG_4);

        return reg_is_null;
}

static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
{
        struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;

        if (!state->initialized) {
                state->initialized = 1;
                state->fit_for_inline = loop_flag_is_zero(env);
                state->callback_subprogno = subprogno;
                return;
        }

        if (!state->fit_for_inline)
                return;

        state->fit_for_inline = (loop_flag_is_zero(env) &&
                                 state->callback_subprogno == subprogno);
}

/* Returns whether or not the given map type can potentially elide
 * lookup return value nullness check. This is possible if the key
 * is statically known.
 */
static bool can_elide_value_nullness(enum bpf_map_type type)
{
        switch (type) {
        case BPF_MAP_TYPE_ARRAY:
        case BPF_MAP_TYPE_PERCPU_ARRAY:
                return true;
        default:
                return false;
        }
}

static int get_helper_proto(struct bpf_verifier_env *env, int func_id,
                            const struct bpf_func_proto **ptr)
{
        if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID)
                return -ERANGE;

        if (!env->ops->get_func_proto)
                return -EINVAL;

        *ptr = env->ops->get_func_proto(func_id, env->prog);
        return *ptr ? 0 : -EINVAL;
}

static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                             int *insn_idx_p)
{
        enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
        bool returns_cpu_specific_alloc_ptr = false;
        const struct bpf_func_proto *fn = NULL;
        enum bpf_return_type ret_type;
        enum bpf_type_flag ret_flag;
        struct bpf_reg_state *regs;
        struct bpf_call_arg_meta meta;
        int insn_idx = *insn_idx_p;
        bool changes_data;
        int i, err, func_id;

        /* find function prototype */
        func_id = insn->imm;
        err = get_helper_proto(env, insn->imm, &fn);
        if (err == -ERANGE) {
                verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id);
                return -EINVAL;
        }

        if (err) {
                verbose(env, "program of this type cannot use helper %s#%d\n",
                        func_id_name(func_id), func_id);
                return err;
        }

        /* eBPF programs must be GPL compatible to use GPL-ed functions */
        if (!env->prog->gpl_compatible && fn->gpl_only) {
                verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
                return -EINVAL;
        }

        if (fn->allowed && !fn->allowed(env->prog)) {
                verbose(env, "helper call is not allowed in probe\n");
                return -EINVAL;
        }

        if (!in_sleepable(env) && fn->might_sleep) {
                verbose(env, "helper call might sleep in a non-sleepable prog\n");
                return -EINVAL;
        }

        /* With LD_ABS/IND some JITs save/restore skb from r1. */
        changes_data = bpf_helper_changes_pkt_data(func_id);
        if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
                verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
                        func_id_name(func_id), func_id);
                return -EINVAL;
        }

        memset(&meta, 0, sizeof(meta));
        meta.pkt_access = fn->pkt_access;

        err = check_func_proto(fn, func_id);
        if (err) {
                verbose(env, "kernel subsystem misconfigured func %s#%d\n",
                        func_id_name(func_id), func_id);
                return err;
        }

        if (env->cur_state->active_rcu_lock) {
                if (fn->might_sleep) {
                        verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",
                                func_id_name(func_id), func_id);
                        return -EINVAL;
                }

                if (in_sleepable(env) && is_storage_get_function(func_id))
                        env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
        }

        if (env->cur_state->active_preempt_locks) {
                if (fn->might_sleep) {
                        verbose(env, "sleepable helper %s#%d in non-preemptible region\n",
                                func_id_name(func_id), func_id);
                        return -EINVAL;
                }

                if (in_sleepable(env) && is_storage_get_function(func_id))
                        env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
        }

        if (env->cur_state->active_irq_id) {
                if (fn->might_sleep) {
                        verbose(env, "sleepable helper %s#%d in IRQ-disabled region\n",
                                func_id_name(func_id), func_id);
                        return -EINVAL;
                }

                if (in_sleepable(env) && is_storage_get_function(func_id))
                        env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
        }

        meta.func_id = func_id;
        /* check args */
        for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
                err = check_func_arg(env, i, &meta, fn, insn_idx);
                if (err)
                        return err;
        }

        err = record_func_map(env, &meta, func_id, insn_idx);
        if (err)
                return err;

        err = record_func_key(env, &meta, func_id, insn_idx);
        if (err)
                return err;

        /* Mark slots with STACK_MISC in case of raw mode, stack offset
         * is inferred from register state.
         */
        for (i = 0; i < meta.access_size; i++) {
                err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
                                       BPF_WRITE, -1, false, false);
                if (err)
                        return err;
        }

        regs = cur_regs(env);

        if (meta.release_regno) {
                err = -EINVAL;
                /* This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot
                 * be released by any dynptr helper. Hence, unmark_stack_slots_dynptr
                 * is safe to do directly.
                 */
                if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) {
                        if (regs[meta.release_regno].type == CONST_PTR_TO_DYNPTR) {
                                verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n");
                                return -EFAULT;
                        }
                        err = unmark_stack_slots_dynptr(env, &regs[meta.release_regno]);
                } else if (func_id == BPF_FUNC_kptr_xchg && meta.ref_obj_id) {
                        u32 ref_obj_id = meta.ref_obj_id;
                        bool in_rcu = in_rcu_cs(env);
                        struct bpf_func_state *state;
                        struct bpf_reg_state *reg;

                        err = release_reference_nomark(env->cur_state, ref_obj_id);
                        if (!err) {
                                bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
                                        if (reg->ref_obj_id == ref_obj_id) {
                                                if (in_rcu && (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) {
                                                        reg->ref_obj_id = 0;
                                                        reg->type &= ~MEM_ALLOC;
                                                        reg->type |= MEM_RCU;
                                                } else {
                                                        mark_reg_invalid(env, reg);
                                                }
                                        }
                                }));
                        }
                } else if (meta.ref_obj_id) {
                        err = release_reference(env, meta.ref_obj_id);
                } else if (register_is_null(&regs[meta.release_regno])) {
                        /* meta.ref_obj_id can only be 0 if register that is meant to be
                         * released is NULL, which must be > R0.
                         */
                        err = 0;
                }
                if (err) {
                        verbose(env, "func %s#%d reference has not been acquired before\n",
                                func_id_name(func_id), func_id);
                        return err;
                }
        }

        switch (func_id) {
        case BPF_FUNC_tail_call:
                err = check_resource_leak(env, false, true, "tail_call");
                if (err)
                        return err;
                break;
        case BPF_FUNC_get_local_storage:
                /* check that flags argument in get_local_storage(map, flags) is 0,
                 * this is required because get_local_storage() can't return an error.
                 */
                if (!register_is_null(&regs[BPF_REG_2])) {
                        verbose(env, "get_local_storage() doesn't support non-zero flags\n");
                        return -EINVAL;
                }
                break;
        case BPF_FUNC_for_each_map_elem:
                err = push_callback_call(env, insn, insn_idx, meta.subprogno,
                                         set_map_elem_callback_state);
                break;
        case BPF_FUNC_timer_set_callback:
                err = push_callback_call(env, insn, insn_idx, meta.subprogno,
                                         set_timer_callback_state);
                break;
        case BPF_FUNC_find_vma:
                err = push_callback_call(env, insn, insn_idx, meta.subprogno,
                                         set_find_vma_callback_state);
                break;
        case BPF_FUNC_snprintf:
                err = check_bpf_snprintf_call(env, regs);
                break;
        case BPF_FUNC_loop:
                update_loop_inline_state(env, meta.subprogno);
                /* Verifier relies on R1 value to determine if bpf_loop() iteration
                 * is finished, thus mark it precise.
                 */
                err = mark_chain_precision(env, BPF_REG_1);
                if (err)
                        return err;
                if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {
                        err = push_callback_call(env, insn, insn_idx, meta.subprogno,
                                                 set_loop_callback_state);
                } else {
                        cur_func(env)->callback_depth = 0;
                        if (env->log.level & BPF_LOG_LEVEL2)
                                verbose(env, "frame%d bpf_loop iteration limit reached\n",
                                        env->cur_state->curframe);
                }
                break;
        case BPF_FUNC_dynptr_from_mem:
                if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) {
                        verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n",
                                reg_type_str(env, regs[BPF_REG_1].type));
                        return -EACCES;
                }
                break;
        case BPF_FUNC_set_retval:
                if (prog_type == BPF_PROG_TYPE_LSM &&
                    env->prog->expected_attach_type == BPF_LSM_CGROUP) {
                        if (!env->prog->aux->attach_func_proto->type) {
                                /* Make sure programs that attach to void
                                 * hooks don't try to modify return value.
                                 */
                                verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
                                return -EINVAL;
                        }
                }
                break;
        case BPF_FUNC_dynptr_data:
        {
                struct bpf_reg_state *reg;
                int id, ref_obj_id;

                reg = get_dynptr_arg_reg(env, fn, regs);
                if (!reg)
                        return -EFAULT;


                if (meta.dynptr_id) {
                        verbose(env, "verifier internal error: meta.dynptr_id already set\n");
                        return -EFAULT;
                }
                if (meta.ref_obj_id) {
                        verbose(env, "verifier internal error: meta.ref_obj_id already set\n");
                        return -EFAULT;
                }

                id = dynptr_id(env, reg);
                if (id < 0) {
                        verbose(env, "verifier internal error: failed to obtain dynptr id\n");
                        return id;
                }

                ref_obj_id = dynptr_ref_obj_id(env, reg);
                if (ref_obj_id < 0) {
                        verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n");
                        return ref_obj_id;
                }

                meta.dynptr_id = id;
                meta.ref_obj_id = ref_obj_id;

                break;
        }
        case BPF_FUNC_dynptr_write:
        {
                enum bpf_dynptr_type dynptr_type;
                struct bpf_reg_state *reg;

                reg = get_dynptr_arg_reg(env, fn, regs);
                if (!reg)
                        return -EFAULT;

                dynptr_type = dynptr_get_type(env, reg);
                if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
                        return -EFAULT;

                if (dynptr_type == BPF_DYNPTR_TYPE_SKB)
                        /* this will trigger clear_all_pkt_pointers(), which will
                         * invalidate all dynptr slices associated with the skb
                         */
                        changes_data = true;

                break;
        }
        case BPF_FUNC_per_cpu_ptr:
        case BPF_FUNC_this_cpu_ptr:
        {
                struct bpf_reg_state *reg = &regs[BPF_REG_1];
                const struct btf_type *type;

                if (reg->type & MEM_RCU) {
                        type = btf_type_by_id(reg->btf, reg->btf_id);
                        if (!type || !btf_type_is_struct(type)) {
                                verbose(env, "Helper has invalid btf/btf_id in R1\n");
                                return -EFAULT;
                        }
                        returns_cpu_specific_alloc_ptr = true;
                        env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true;
                }
                break;
        }
        case BPF_FUNC_user_ringbuf_drain:
                err = push_callback_call(env, insn, insn_idx, meta.subprogno,
                                         set_user_ringbuf_callback_state);
                break;
        }

        if (err)
                return err;

        /* reset caller saved regs */
        for (i = 0; i < CALLER_SAVED_REGS; i++) {
                mark_reg_not_init(env, regs, caller_saved[i]);
                check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
        }

        /* helper call returns 64-bit value. */
        regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;

        /* update return register (already marked as written above) */
        ret_type = fn->ret_type;
        ret_flag = type_flag(ret_type);

        switch (base_type(ret_type)) {
        case RET_INTEGER:
                /* sets type to SCALAR_VALUE */
                mark_reg_unknown(env, regs, BPF_REG_0);
                break;
        case RET_VOID:
                regs[BPF_REG_0].type = NOT_INIT;
                break;
        case RET_PTR_TO_MAP_VALUE:
                /* There is no offset yet applied, variable or fixed */
                mark_reg_known_zero(env, regs, BPF_REG_0);
                /* remember map_ptr, so that check_map_access()
                 * can check 'value_size' boundary of memory access
                 * to map element returned from bpf_map_lookup_elem()
                 */
                if (meta.map_ptr == NULL) {
                        verbose(env,
                                "kernel subsystem misconfigured verifier\n");
                        return -EINVAL;
                }

                if (func_id == BPF_FUNC_map_lookup_elem &&
                    can_elide_value_nullness(meta.map_ptr->map_type) &&
                    meta.const_map_key >= 0 &&
                    meta.const_map_key < meta.map_ptr->max_entries)
                        ret_flag &= ~PTR_MAYBE_NULL;

                regs[BPF_REG_0].map_ptr = meta.map_ptr;
                regs[BPF_REG_0].map_uid = meta.map_uid;
                regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
                if (!type_may_be_null(ret_flag) &&
                    btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) {
                        regs[BPF_REG_0].id = ++env->id_gen;
                }
                break;
        case RET_PTR_TO_SOCKET:
                mark_reg_known_zero(env, regs, BPF_REG_0);
                regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag;
                break;
        case RET_PTR_TO_SOCK_COMMON:
                mark_reg_known_zero(env, regs, BPF_REG_0);
                regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag;
                break;
        case RET_PTR_TO_TCP_SOCK:
                mark_reg_known_zero(env, regs, BPF_REG_0);
                regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag;
                break;
        case RET_PTR_TO_MEM:
                mark_reg_known_zero(env, regs, BPF_REG_0);
                regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
                regs[BPF_REG_0].mem_size = meta.mem_size;
                break;
        case RET_PTR_TO_MEM_OR_BTF_ID:
        {
                const struct btf_type *t;

                mark_reg_known_zero(env, regs, BPF_REG_0);
                t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL);
                if (!btf_type_is_struct(t)) {
                        u32 tsize;
                        const struct btf_type *ret;
                        const char *tname;

                        /* resolve the type size of ksym. */
                        ret = btf_resolve_size(meta.ret_btf, t, &tsize);
                        if (IS_ERR(ret)) {
                                tname = btf_name_by_offset(meta.ret_btf, t->name_off);
                                verbose(env, "unable to resolve the size of type '%s': %ld\n",
                                        tname, PTR_ERR(ret));
                                return -EINVAL;
                        }
                        regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
                        regs[BPF_REG_0].mem_size = tsize;
                } else {
                        if (returns_cpu_specific_alloc_ptr) {
                                regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU;
                        } else {
                                /* MEM_RDONLY may be carried from ret_flag, but it
                                 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
                                 * it will confuse the check of PTR_TO_BTF_ID in
                                 * check_mem_access().
                                 */
                                ret_flag &= ~MEM_RDONLY;
                                regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
                        }

                        regs[BPF_REG_0].btf = meta.ret_btf;
                        regs[BPF_REG_0].btf_id = meta.ret_btf_id;
                }
                break;
        }
        case RET_PTR_TO_BTF_ID:
        {
                struct btf *ret_btf;
                int ret_btf_id;

                mark_reg_known_zero(env, regs, BPF_REG_0);
                regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
                if (func_id == BPF_FUNC_kptr_xchg) {
                        ret_btf = meta.kptr_field->kptr.btf;
                        ret_btf_id = meta.kptr_field->kptr.btf_id;
                        if (!btf_is_kernel(ret_btf)) {
                                regs[BPF_REG_0].type |= MEM_ALLOC;
                                if (meta.kptr_field->type == BPF_KPTR_PERCPU)
                                        regs[BPF_REG_0].type |= MEM_PERCPU;
                        }
                } else {
                        if (fn->ret_btf_id == BPF_PTR_POISON) {
                                verbose(env, "verifier internal error:");
                                verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n",
                                        func_id_name(func_id));
                                return -EINVAL;
                        }
                        ret_btf = btf_vmlinux;
                        ret_btf_id = *fn->ret_btf_id;
                }
                if (ret_btf_id == 0) {
                        verbose(env, "invalid return type %u of func %s#%d\n",
                                base_type(ret_type), func_id_name(func_id),
                                func_id);
                        return -EINVAL;
                }
                regs[BPF_REG_0].btf = ret_btf;
                regs[BPF_REG_0].btf_id = ret_btf_id;
                break;
        }
        default:
                verbose(env, "unknown return type %u of func %s#%d\n",
                        base_type(ret_type), func_id_name(func_id), func_id);
                return -EINVAL;
        }

        if (type_may_be_null(regs[BPF_REG_0].type))
                regs[BPF_REG_0].id = ++env->id_gen;

        if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) {
                verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n",
                        func_id_name(func_id), func_id);
                return -EFAULT;
        }

        if (is_dynptr_ref_function(func_id))
                regs[BPF_REG_0].dynptr_id = meta.dynptr_id;

        if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) {
                /* For release_reference() */
                regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
        } else if (is_acquire_function(func_id, meta.map_ptr)) {
                int id = acquire_reference(env, insn_idx);

                if (id < 0)
                        return id;
                /* For mark_ptr_or_null_reg() */
                regs[BPF_REG_0].id = id;
                /* For release_reference() */
                regs[BPF_REG_0].ref_obj_id = id;
        }

        err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta);
        if (err)
                return err;

        err = check_map_func_compatibility(env, meta.map_ptr, func_id);
        if (err)
                return err;

        if ((func_id == BPF_FUNC_get_stack ||
             func_id == BPF_FUNC_get_task_stack) &&
            !env->prog->has_callchain_buf) {
                const char *err_str;

#ifdef CONFIG_PERF_EVENTS
                err = get_callchain_buffers(sysctl_perf_event_max_stack);
                err_str = "cannot get callchain buffer for func %s#%d\n";
#else
                err = -ENOTSUPP;
                err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
#endif
                if (err) {
                        verbose(env, err_str, func_id_name(func_id), func_id);
                        return err;
                }

                env->prog->has_callchain_buf = true;
        }

        if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack)
                env->prog->call_get_stack = true;

        if (func_id == BPF_FUNC_get_func_ip) {
                if (check_get_func_ip(env))
                        return -ENOTSUPP;
                env->prog->call_get_func_ip = true;
        }

        if (changes_data)
                clear_all_pkt_pointers(env);
        return 0;
}

/* mark_btf_func_reg_size() is used when the reg size is determined by
 * the BTF func_proto's return value size and argument.
 */
static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
                                   size_t reg_size)
{
        struct bpf_reg_state *reg = &cur_regs(env)[regno];

        if (regno == BPF_REG_0) {
                /* Function return value */
                reg->live |= REG_LIVE_WRITTEN;
                reg->subreg_def = reg_size == sizeof(u64) ?
                        DEF_NOT_SUBREG : env->insn_idx + 1;
        } else {
                /* Function argument */
                if (reg_size == sizeof(u64)) {
                        mark_insn_zext(env, reg);
                        mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
                } else {
                        mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
                }
        }
}

static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_ACQUIRE;
}

static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_RELEASE;
}

static bool is_kfunc_trusted_args(struct bpf_kfunc_call_arg_meta *meta)
{
        return (meta->kfunc_flags & KF_TRUSTED_ARGS) || is_kfunc_release(meta);
}

static bool is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_SLEEPABLE;
}

static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_DESTRUCTIVE;
}

static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_RCU;
}

static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_RCU_PROTECTED;
}

static bool is_kfunc_arg_mem_size(const struct btf *btf,
                                  const struct btf_param *arg,
                                  const struct bpf_reg_state *reg)
{
        const struct btf_type *t;

        t = btf_type_skip_modifiers(btf, arg->type, NULL);
        if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
                return false;

        return btf_param_match_suffix(btf, arg, "__sz");
}

static bool is_kfunc_arg_const_mem_size(const struct btf *btf,
                                        const struct btf_param *arg,
                                        const struct bpf_reg_state *reg)
{
        const struct btf_type *t;

        t = btf_type_skip_modifiers(btf, arg->type, NULL);
        if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
                return false;

        return btf_param_match_suffix(btf, arg, "__szk");
}

static bool is_kfunc_arg_optional(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__opt");
}

static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__k");
}

static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__ign");
}

static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__map");
}

static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__alloc");
}

static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__uninit");
}

static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__refcounted_kptr");
}

static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__nullable");
}

static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__str");
}

static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param *arg)
{
        return btf_param_match_suffix(btf, arg, "__irq_flag");
}

static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
                                          const struct btf_param *arg,
                                          const char *name)
{
        int len, target_len = strlen(name);
        const char *param_name;

        param_name = btf_name_by_offset(btf, arg->name_off);
        if (str_is_empty(param_name))
                return false;
        len = strlen(param_name);
        if (len != target_len)
                return false;
        if (strcmp(param_name, name))
                return false;

        return true;
}

enum {
        KF_ARG_DYNPTR_ID,
        KF_ARG_LIST_HEAD_ID,
        KF_ARG_LIST_NODE_ID,
        KF_ARG_RB_ROOT_ID,
        KF_ARG_RB_NODE_ID,
        KF_ARG_WORKQUEUE_ID,
};

BTF_ID_LIST(kf_arg_btf_ids)
BTF_ID(struct, bpf_dynptr)
BTF_ID(struct, bpf_list_head)
BTF_ID(struct, bpf_list_node)
BTF_ID(struct, bpf_rb_root)
BTF_ID(struct, bpf_rb_node)
BTF_ID(struct, bpf_wq)

static bool __is_kfunc_ptr_arg_type(const struct btf *btf,
                                    const struct btf_param *arg, int type)
{
        const struct btf_type *t;
        u32 res_id;

        t = btf_type_skip_modifiers(btf, arg->type, NULL);
        if (!t)
                return false;
        if (!btf_type_is_ptr(t))
                return false;
        t = btf_type_skip_modifiers(btf, t->type, &res_id);
        if (!t)
                return false;
        return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]);
}

static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg)
{
        return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID);
}

static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg)
{
        return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID);
}

static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg)
{
        return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID);
}

static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg)
{
        return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID);
}

static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg)
{
        return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID);
}

static bool is_kfunc_arg_wq(const struct btf *btf, const struct btf_param *arg)
{
        return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_WORKQUEUE_ID);
}

static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf,
                                  const struct btf_param *arg)
{
        const struct btf_type *t;

        t = btf_type_resolve_func_ptr(btf, arg->type, NULL);
        if (!t)
                return false;

        return true;
}

/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */
static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,
                                        const struct btf *btf,
                                        const struct btf_type *t, int rec)
{
        const struct btf_type *member_type;
        const struct btf_member *member;
        u32 i;

        if (!btf_type_is_struct(t))
                return false;

        for_each_member(i, t, member) {
                const struct btf_array *array;

                member_type = btf_type_skip_modifiers(btf, member->type, NULL);
                if (btf_type_is_struct(member_type)) {
                        if (rec >= 3) {
                                verbose(env, "max struct nesting depth exceeded\n");
                                return false;
                        }
                        if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1))
                                return false;
                        continue;
                }
                if (btf_type_is_array(member_type)) {
                        array = btf_array(member_type);
                        if (!array->nelems)
                                return false;
                        member_type = btf_type_skip_modifiers(btf, array->type, NULL);
                        if (!btf_type_is_scalar(member_type))
                                return false;
                        continue;
                }
                if (!btf_type_is_scalar(member_type))
                        return false;
        }
        return true;
}

enum kfunc_ptr_arg_type {
        KF_ARG_PTR_TO_CTX,
        KF_ARG_PTR_TO_ALLOC_BTF_ID,    /* Allocated object */
        KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */
        KF_ARG_PTR_TO_DYNPTR,
        KF_ARG_PTR_TO_ITER,
        KF_ARG_PTR_TO_LIST_HEAD,
        KF_ARG_PTR_TO_LIST_NODE,
        KF_ARG_PTR_TO_BTF_ID,               /* Also covers reg2btf_ids conversions */
        KF_ARG_PTR_TO_MEM,
        KF_ARG_PTR_TO_MEM_SIZE,               /* Size derived from next argument, skip it */
        KF_ARG_PTR_TO_CALLBACK,
        KF_ARG_PTR_TO_RB_ROOT,
        KF_ARG_PTR_TO_RB_NODE,
        KF_ARG_PTR_TO_NULL,
        KF_ARG_PTR_TO_CONST_STR,
        KF_ARG_PTR_TO_MAP,
        KF_ARG_PTR_TO_WORKQUEUE,
        KF_ARG_PTR_TO_IRQ_FLAG,
};

enum special_kfunc_type {
        KF_bpf_obj_new_impl,
        KF_bpf_obj_drop_impl,
        KF_bpf_refcount_acquire_impl,
        KF_bpf_list_push_front_impl,
        KF_bpf_list_push_back_impl,
        KF_bpf_list_pop_front,
        KF_bpf_list_pop_back,
        KF_bpf_cast_to_kern_ctx,
        KF_bpf_rdonly_cast,
        KF_bpf_rcu_read_lock,
        KF_bpf_rcu_read_unlock,
        KF_bpf_rbtree_remove,
        KF_bpf_rbtree_add_impl,
        KF_bpf_rbtree_first,
        KF_bpf_dynptr_from_skb,
        KF_bpf_dynptr_from_xdp,
        KF_bpf_dynptr_slice,
        KF_bpf_dynptr_slice_rdwr,
        KF_bpf_dynptr_clone,
        KF_bpf_percpu_obj_new_impl,
        KF_bpf_percpu_obj_drop_impl,
        KF_bpf_throw,
        KF_bpf_wq_set_callback_impl,
        KF_bpf_preempt_disable,
        KF_bpf_preempt_enable,
        KF_bpf_iter_css_task_new,
        KF_bpf_session_cookie,
        KF_bpf_get_kmem_cache,
        KF_bpf_local_irq_save,
        KF_bpf_local_irq_restore,
        KF_bpf_iter_num_new,
        KF_bpf_iter_num_next,
        KF_bpf_iter_num_destroy,
};

BTF_SET_START(special_kfunc_set)
BTF_ID(func, bpf_obj_new_impl)
BTF_ID(func, bpf_obj_drop_impl)
BTF_ID(func, bpf_refcount_acquire_impl)
BTF_ID(func, bpf_list_push_front_impl)
BTF_ID(func, bpf_list_push_back_impl)
BTF_ID(func, bpf_list_pop_front)
BTF_ID(func, bpf_list_pop_back)
BTF_ID(func, bpf_cast_to_kern_ctx)
BTF_ID(func, bpf_rdonly_cast)
BTF_ID(func, bpf_rbtree_remove)
BTF_ID(func, bpf_rbtree_add_impl)
BTF_ID(func, bpf_rbtree_first)
#ifdef CONFIG_NET
BTF_ID(func, bpf_dynptr_from_skb)
BTF_ID(func, bpf_dynptr_from_xdp)
#endif
BTF_ID(func, bpf_dynptr_slice)
BTF_ID(func, bpf_dynptr_slice_rdwr)
BTF_ID(func, bpf_dynptr_clone)
BTF_ID(func, bpf_percpu_obj_new_impl)
BTF_ID(func, bpf_percpu_obj_drop_impl)
BTF_ID(func, bpf_throw)
BTF_ID(func, bpf_wq_set_callback_impl)
#ifdef CONFIG_CGROUPS
BTF_ID(func, bpf_iter_css_task_new)
#endif
BTF_SET_END(special_kfunc_set)

BTF_ID_LIST(special_kfunc_list)
BTF_ID(func, bpf_obj_new_impl)
BTF_ID(func, bpf_obj_drop_impl)
BTF_ID(func, bpf_refcount_acquire_impl)
BTF_ID(func, bpf_list_push_front_impl)
BTF_ID(func, bpf_list_push_back_impl)
BTF_ID(func, bpf_list_pop_front)
BTF_ID(func, bpf_list_pop_back)
BTF_ID(func, bpf_cast_to_kern_ctx)
BTF_ID(func, bpf_rdonly_cast)
BTF_ID(func, bpf_rcu_read_lock)
BTF_ID(func, bpf_rcu_read_unlock)
BTF_ID(func, bpf_rbtree_remove)
BTF_ID(func, bpf_rbtree_add_impl)
BTF_ID(func, bpf_rbtree_first)
#ifdef CONFIG_NET
BTF_ID(func, bpf_dynptr_from_skb)
BTF_ID(func, bpf_dynptr_from_xdp)
#else
BTF_ID_UNUSED
BTF_ID_UNUSED
#endif
BTF_ID(func, bpf_dynptr_slice)
BTF_ID(func, bpf_dynptr_slice_rdwr)
BTF_ID(func, bpf_dynptr_clone)
BTF_ID(func, bpf_percpu_obj_new_impl)
BTF_ID(func, bpf_percpu_obj_drop_impl)
BTF_ID(func, bpf_throw)
BTF_ID(func, bpf_wq_set_callback_impl)
BTF_ID(func, bpf_preempt_disable)
BTF_ID(func, bpf_preempt_enable)
#ifdef CONFIG_CGROUPS
BTF_ID(func, bpf_iter_css_task_new)
#else
BTF_ID_UNUSED
#endif
#ifdef CONFIG_BPF_EVENTS
BTF_ID(func, bpf_session_cookie)
#else
BTF_ID_UNUSED
#endif
BTF_ID(func, bpf_get_kmem_cache)
BTF_ID(func, bpf_local_irq_save)
BTF_ID(func, bpf_local_irq_restore)
BTF_ID(func, bpf_iter_num_new)
BTF_ID(func, bpf_iter_num_next)
BTF_ID(func, bpf_iter_num_destroy)

static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
{
        if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&
            meta->arg_owning_ref) {
                return false;
        }

        return meta->kfunc_flags & KF_RET_NULL;
}

static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock];
}

static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock];
}

static bool is_kfunc_bpf_preempt_disable(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable];
}

static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable];
}

static enum kfunc_ptr_arg_type
get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
                       struct bpf_kfunc_call_arg_meta *meta,
                       const struct btf_type *t, const struct btf_type *ref_t,
                       const char *ref_tname, const struct btf_param *args,
                       int argno, int nargs)
{
        u32 regno = argno + 1;
        struct bpf_reg_state *regs = cur_regs(env);
        struct bpf_reg_state *reg = &regs[regno];
        bool arg_mem_size = false;

        if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx])
                return KF_ARG_PTR_TO_CTX;

        /* In this function, we verify the kfunc's BTF as per the argument type,
         * leaving the rest of the verification with respect to the register
         * type to our caller. When a set of conditions hold in the BTF type of
         * arguments, we resolve it to a known kfunc_ptr_arg_type.
         */
        if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno))
                return KF_ARG_PTR_TO_CTX;

        if (is_kfunc_arg_nullable(meta->btf, &args[argno]) && register_is_null(reg))
                return KF_ARG_PTR_TO_NULL;

        if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_ALLOC_BTF_ID;

        if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_REFCOUNTED_KPTR;

        if (is_kfunc_arg_dynptr(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_DYNPTR;

        if (is_kfunc_arg_iter(meta, argno, &args[argno]))
                return KF_ARG_PTR_TO_ITER;

        if (is_kfunc_arg_list_head(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_LIST_HEAD;

        if (is_kfunc_arg_list_node(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_LIST_NODE;

        if (is_kfunc_arg_rbtree_root(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_RB_ROOT;

        if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_RB_NODE;

        if (is_kfunc_arg_const_str(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_CONST_STR;

        if (is_kfunc_arg_map(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_MAP;

        if (is_kfunc_arg_wq(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_WORKQUEUE;

        if (is_kfunc_arg_irq_flag(meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_IRQ_FLAG;

        if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) {
                if (!btf_type_is_struct(ref_t)) {
                        verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n",
                                meta->func_name, argno, btf_type_str(ref_t), ref_tname);
                        return -EINVAL;
                }
                return KF_ARG_PTR_TO_BTF_ID;
        }

        if (is_kfunc_arg_callback(env, meta->btf, &args[argno]))
                return KF_ARG_PTR_TO_CALLBACK;

        if (argno + 1 < nargs &&
            (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1]) ||
             is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1])))
                arg_mem_size = true;

        /* This is the catch all argument type of register types supported by
         * check_helper_mem_access. However, we only allow when argument type is
         * pointer to scalar, or struct composed (recursively) of scalars. When
         * arg_mem_size is true, the pointer can be void *.
         */
        if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) &&
            (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) {
                verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n",
                        argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : "");
                return -EINVAL;
        }
        return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM;
}

static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
                                        struct bpf_reg_state *reg,
                                        const struct btf_type *ref_t,
                                        const char *ref_tname, u32 ref_id,
                                        struct bpf_kfunc_call_arg_meta *meta,
                                        int argno)
{
        const struct btf_type *reg_ref_t;
        bool strict_type_match = false;
        const struct btf *reg_btf;
        const char *reg_ref_tname;
        bool taking_projection;
        bool struct_same;
        u32 reg_ref_id;

        if (base_type(reg->type) == PTR_TO_BTF_ID) {
                reg_btf = reg->btf;
                reg_ref_id = reg->btf_id;
        } else {
                reg_btf = btf_vmlinux;
                reg_ref_id = *reg2btf_ids[base_type(reg->type)];
        }

        /* Enforce strict type matching for calls to kfuncs that are acquiring
         * or releasing a reference, or are no-cast aliases. We do _not_
         * enforce strict matching for plain KF_TRUSTED_ARGS kfuncs by default,
         * as we want to enable BPF programs to pass types that are bitwise
         * equivalent without forcing them to explicitly cast with something
         * like bpf_cast_to_kern_ctx().
         *
         * For example, say we had a type like the following:
         *
         * struct bpf_cpumask {
         *        cpumask_t cpumask;
         *        refcount_t usage;
         * };
         *
         * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed
         * to a struct cpumask, so it would be safe to pass a struct
         * bpf_cpumask * to a kfunc expecting a struct cpumask *.
         *
         * The philosophy here is similar to how we allow scalars of different
         * types to be passed to kfuncs as long as the size is the same. The
         * only difference here is that we're simply allowing
         * btf_struct_ids_match() to walk the struct at the 0th offset, and
         * resolve types.
         */
        if ((is_kfunc_release(meta) && reg->ref_obj_id) ||
            btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))
                strict_type_match = true;

        WARN_ON_ONCE(is_kfunc_release(meta) &&
                     (reg->off || !tnum_is_const(reg->var_off) ||
                      reg->var_off.value));

        reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, &reg_ref_id);
        reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off);
        struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match);
        /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
         * actually use it -- it must cast to the underlying type. So we allow
         * caller to pass in the underlying type.
         */
        taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname);
        if (!taking_projection && !struct_same) {
                verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
                        meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1,
                        btf_type_str(reg_ref_t), reg_ref_tname);
                return -EINVAL;
        }
        return 0;
}

static int process_irq_flag(struct bpf_verifier_env *env, int regno,
                             struct bpf_kfunc_call_arg_meta *meta)
{
        struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
        bool irq_save;
        int err;

        if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save]) {
                irq_save = true;
        } else if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_restore]) {
                irq_save = false;
        } else {
                verbose(env, "verifier internal error: unknown irq flags kfunc\n");
                return -EFAULT;
        }

        if (irq_save) {
                if (!is_irq_flag_reg_valid_uninit(env, reg)) {
                        verbose(env, "expected uninitialized irq flag as arg#%d\n", regno - 1);
                        return -EINVAL;
                }

                err = check_mem_access(env, env->insn_idx, regno, 0, BPF_DW, BPF_WRITE, -1, false, false);
                if (err)
                        return err;

                err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx);
                if (err)
                        return err;
        } else {
                err = is_irq_flag_reg_valid_init(env, reg);
                if (err) {
                        verbose(env, "expected an initialized irq flag as arg#%d\n", regno - 1);
                        return err;
                }

                err = mark_irq_flag_read(env, reg);
                if (err)
                        return err;

                err = unmark_stack_slot_irq_flag(env, reg);
                if (err)
                        return err;
        }
        return 0;
}


static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        struct btf_record *rec = reg_btf_record(reg);

        if (!env->cur_state->active_locks) {
                verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n");
                return -EFAULT;
        }

        if (type_flag(reg->type) & NON_OWN_REF) {
                verbose(env, "verifier internal error: NON_OWN_REF already set\n");
                return -EFAULT;
        }

        reg->type |= NON_OWN_REF;
        if (rec->refcount_off >= 0)
                reg->type |= MEM_RCU;

        return 0;
}

static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id)
{
        struct bpf_verifier_state *state = env->cur_state;
        struct bpf_func_state *unused;
        struct bpf_reg_state *reg;
        int i;

        if (!ref_obj_id) {
                verbose(env, "verifier internal error: ref_obj_id is zero for "
                             "owning -> non-owning conversion\n");
                return -EFAULT;
        }

        for (i = 0; i < state->acquired_refs; i++) {
                if (state->refs[i].id != ref_obj_id)
                        continue;

                /* Clear ref_obj_id here so release_reference doesn't clobber
                 * the whole reg
                 */
                bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
                        if (reg->ref_obj_id == ref_obj_id) {
                                reg->ref_obj_id = 0;
                                ref_set_non_owning(env, reg);
                        }
                }));
                return 0;
        }

        verbose(env, "verifier internal error: ref state missing for ref_obj_id\n");
        return -EFAULT;
}

/* Implementation details:
 *
 * Each register points to some region of memory, which we define as an
 * allocation. Each allocation may embed a bpf_spin_lock which protects any
 * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same
 * allocation. The lock and the data it protects are colocated in the same
 * memory region.
 *
 * Hence, everytime a register holds a pointer value pointing to such
 * allocation, the verifier preserves a unique reg->id for it.
 *
 * The verifier remembers the lock 'ptr' and the lock 'id' whenever
 * bpf_spin_lock is called.
 *
 * To enable this, lock state in the verifier captures two values:
 *        active_lock.ptr = Register's type specific pointer
 *        active_lock.id  = A unique ID for each register pointer value
 *
 * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two
 * supported register types.
 *
 * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of
 * allocated objects is the reg->btf pointer.
 *
 * The active_lock.id is non-unique for maps supporting direct_value_addr, as we
 * can establish the provenance of the map value statically for each distinct
 * lookup into such maps. They always contain a single map value hence unique
 * IDs for each pseudo load pessimizes the algorithm and rejects valid programs.
 *
 * So, in case of global variables, they use array maps with max_entries = 1,
 * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point
 * into the same map value as max_entries is 1, as described above).
 *
 * In case of inner map lookups, the inner map pointer has same map_ptr as the
 * outer map pointer (in verifier context), but each lookup into an inner map
 * assigns a fresh reg->id to the lookup, so while lookups into distinct inner
 * maps from the same outer map share the same map_ptr as active_lock.ptr, they
 * will get different reg->id assigned to each lookup, hence different
 * active_lock.id.
 *
 * In case of allocated objects, active_lock.ptr is the reg->btf, and the
 * reg->id is a unique ID preserved after the NULL pointer check on the pointer
 * returned from bpf_obj_new. Each allocation receives a new reg->id.
 */
static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
        struct bpf_reference_state *s;
        void *ptr;
        u32 id;

        switch ((int)reg->type) {
        case PTR_TO_MAP_VALUE:
                ptr = reg->map_ptr;
                break;
        case PTR_TO_BTF_ID | MEM_ALLOC:
                ptr = reg->btf;
                break;
        default:
                verbose(env, "verifier internal error: unknown reg type for lock check\n");
                return -EFAULT;
        }
        id = reg->id;

        if (!env->cur_state->active_locks)
                return -EINVAL;
        s = find_lock_state(env->cur_state, REF_TYPE_LOCK, id, ptr);
        if (!s) {
                verbose(env, "held lock and object are not in the same allocation\n");
                return -EINVAL;
        }
        return 0;
}

static bool is_bpf_list_api_kfunc(u32 btf_id)
{
        return btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
               btf_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
               btf_id == special_kfunc_list[KF_bpf_list_pop_front] ||
               btf_id == special_kfunc_list[KF_bpf_list_pop_back];
}

static bool is_bpf_rbtree_api_kfunc(u32 btf_id)
{
        return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] ||
               btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
               btf_id == special_kfunc_list[KF_bpf_rbtree_first];
}

static bool is_bpf_iter_num_api_kfunc(u32 btf_id)
{
        return btf_id == special_kfunc_list[KF_bpf_iter_num_new] ||
               btf_id == special_kfunc_list[KF_bpf_iter_num_next] ||
               btf_id == special_kfunc_list[KF_bpf_iter_num_destroy];
}

static bool is_bpf_graph_api_kfunc(u32 btf_id)
{
        return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id) ||
               btf_id == special_kfunc_list[KF_bpf_refcount_acquire_impl];
}

static bool kfunc_spin_allowed(u32 btf_id)
{
        return is_bpf_graph_api_kfunc(btf_id) || is_bpf_iter_num_api_kfunc(btf_id);
}

static bool is_sync_callback_calling_kfunc(u32 btf_id)
{
        return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl];
}

static bool is_async_callback_calling_kfunc(u32 btf_id)
{
        return btf_id == special_kfunc_list[KF_bpf_wq_set_callback_impl];
}

static bool is_bpf_throw_kfunc(struct bpf_insn *insn)
{
        return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
               insn->imm == special_kfunc_list[KF_bpf_throw];
}

static bool is_bpf_wq_set_callback_impl_kfunc(u32 btf_id)
{
        return btf_id == special_kfunc_list[KF_bpf_wq_set_callback_impl];
}

static bool is_callback_calling_kfunc(u32 btf_id)
{
        return is_sync_callback_calling_kfunc(btf_id) ||
               is_async_callback_calling_kfunc(btf_id);
}

static bool is_rbtree_lock_required_kfunc(u32 btf_id)
{
        return is_bpf_rbtree_api_kfunc(btf_id);
}

static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env,
                                          enum btf_field_type head_field_type,
                                          u32 kfunc_btf_id)
{
        bool ret;

        switch (head_field_type) {
        case BPF_LIST_HEAD:
                ret = is_bpf_list_api_kfunc(kfunc_btf_id);
                break;
        case BPF_RB_ROOT:
                ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id);
                break;
        default:
                verbose(env, "verifier internal error: unexpected graph root argument type %s\n",
                        btf_field_type_name(head_field_type));
                return false;
        }

        if (!ret)
                verbose(env, "verifier internal error: %s head arg for unknown kfunc\n",
                        btf_field_type_name(head_field_type));
        return ret;
}

static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,
                                          enum btf_field_type node_field_type,
                                          u32 kfunc_btf_id)
{
        bool ret;

        switch (node_field_type) {
        case BPF_LIST_NODE:
                ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
                       kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_back_impl]);
                break;
        case BPF_RB_NODE:
                ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
                       kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]);
                break;
        default:
                verbose(env, "verifier internal error: unexpected graph node argument type %s\n",
                        btf_field_type_name(node_field_type));
                return false;
        }

        if (!ret)
                verbose(env, "verifier internal error: %s node arg for unknown kfunc\n",
                        btf_field_type_name(node_field_type));
        return ret;
}

static int
__process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
                                   struct bpf_reg_state *reg, u32 regno,
                                   struct bpf_kfunc_call_arg_meta *meta,
                                   enum btf_field_type head_field_type,
                                   struct btf_field **head_field)
{
        const char *head_type_name;
        struct btf_field *field;
        struct btf_record *rec;
        u32 head_off;

        if (meta->btf != btf_vmlinux) {
                verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n");
                return -EFAULT;
        }

        if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id))
                return -EFAULT;

        head_type_name = btf_field_type_name(head_field_type);
        if (!tnum_is_const(reg->var_off)) {
                verbose(env,
                        "R%d doesn't have constant offset. %s has to be at the constant offset\n",
                        regno, head_type_name);
                return -EINVAL;
        }

        rec = reg_btf_record(reg);
        head_off = reg->off + reg->var_off.value;
        field = btf_record_find(rec, head_off, head_field_type);
        if (!field) {
                verbose(env, "%s not found at offset=%u\n", head_type_name, head_off);
                return -EINVAL;
        }

        /* All functions require bpf_list_head to be protected using a bpf_spin_lock */
        if (check_reg_allocation_locked(env, reg)) {
                verbose(env, "bpf_spin_lock at off=%d must be held for %s\n",
                        rec->spin_lock_off, head_type_name);
                return -EINVAL;
        }

        if (*head_field) {
                verbose(env, "verifier internal error: repeating %s arg\n", head_type_name);
                return -EFAULT;
        }
        *head_field = field;
        return 0;
}

static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
                                           struct bpf_reg_state *reg, u32 regno,
                                           struct bpf_kfunc_call_arg_meta *meta)
{
        return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD,
                                                          &meta->arg_list_head.field);
}

static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
                                             struct bpf_reg_state *reg, u32 regno,
                                             struct bpf_kfunc_call_arg_meta *meta)
{
        return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT,
                                                          &meta->arg_rbtree_root.field);
}

static int
__process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
                                   struct bpf_reg_state *reg, u32 regno,
                                   struct bpf_kfunc_call_arg_meta *meta,
                                   enum btf_field_type head_field_type,
                                   enum btf_field_type node_field_type,
                                   struct btf_field **node_field)
{
        const char *node_type_name;
        const struct btf_type *et, *t;
        struct btf_field *field;
        u32 node_off;

        if (meta->btf != btf_vmlinux) {
                verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n");
                return -EFAULT;
        }

        if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id))
                return -EFAULT;

        node_type_name = btf_field_type_name(node_field_type);
        if (!tnum_is_const(reg->var_off)) {
                verbose(env,
                        "R%d doesn't have constant offset. %s has to be at the constant offset\n",
                        regno, node_type_name);
                return -EINVAL;
        }

        node_off = reg->off + reg->var_off.value;
        field = reg_find_field_offset(reg, node_off, node_field_type);
        if (!field) {
                verbose(env, "%s not found at offset=%u\n", node_type_name, node_off);
                return -EINVAL;
        }

        field = *node_field;

        et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id);
        t = btf_type_by_id(reg->btf, reg->btf_id);
        if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,
                                  field->graph_root.value_btf_id, true)) {
                verbose(env, "operation on %s expects arg#1 %s at offset=%d "
                        "in struct %s, but arg is at offset=%d in struct %s\n",
                        btf_field_type_name(head_field_type),
                        btf_field_type_name(node_field_type),
                        field->graph_root.node_offset,
                        btf_name_by_offset(field->graph_root.btf, et->name_off),
                        node_off, btf_name_by_offset(reg->btf, t->name_off));
                return -EINVAL;
        }
        meta->arg_btf = reg->btf;
        meta->arg_btf_id = reg->btf_id;

        if (node_off != field->graph_root.node_offset) {
                verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",
                        node_off, btf_field_type_name(node_field_type),
                        field->graph_root.node_offset,
                        btf_name_by_offset(field->graph_root.btf, et->name_off));
                return -EINVAL;
        }

        return 0;
}

static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
                                           struct bpf_reg_state *reg, u32 regno,
                                           struct bpf_kfunc_call_arg_meta *meta)
{
        return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
                                                  BPF_LIST_HEAD, BPF_LIST_NODE,
                                                  &meta->arg_list_head.field);
}

static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
                                             struct bpf_reg_state *reg, u32 regno,
                                             struct bpf_kfunc_call_arg_meta *meta)
{
        return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
                                                  BPF_RB_ROOT, BPF_RB_NODE,
                                                  &meta->arg_rbtree_root.field);
}

/*
 * css_task iter allowlist is needed to avoid dead locking on css_set_lock.
 * LSM hooks and iters (both sleepable and non-sleepable) are safe.
 * Any sleepable progs are also safe since bpf_check_attach_target() enforce
 * them can only be attached to some specific hook points.
 */
static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)
{
        enum bpf_prog_type prog_type = resolve_prog_type(env->prog);

        switch (prog_type) {
        case BPF_PROG_TYPE_LSM:
                return true;
        case BPF_PROG_TYPE_TRACING:
                if (env->prog->expected_attach_type == BPF_TRACE_ITER)
                        return true;
                fallthrough;
        default:
                return in_sleepable(env);
        }
}

static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,
                            int insn_idx)
{
        const char *func_name = meta->func_name, *ref_tname;
        const struct btf *btf = meta->btf;
        const struct btf_param *args;
        struct btf_record *rec;
        u32 i, nargs;
        int ret;

        args = (const struct btf_param *)(meta->func_proto + 1);
        nargs = btf_type_vlen(meta->func_proto);
        if (nargs > MAX_BPF_FUNC_REG_ARGS) {
                verbose(env, "Function %s has %d > %d args\n", func_name, nargs,
                        MAX_BPF_FUNC_REG_ARGS);
                return -EINVAL;
        }

        /* Check that BTF function arguments match actual types that the
         * verifier sees.
         */
        for (i = 0; i < nargs; i++) {
                struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[i + 1];
                const struct btf_type *t, *ref_t, *resolve_ret;
                enum bpf_arg_type arg_type = ARG_DONTCARE;
                u32 regno = i + 1, ref_id, type_size;
                bool is_ret_buf_sz = false;
                int kf_arg_type;

                t = btf_type_skip_modifiers(btf, args[i].type, NULL);

                if (is_kfunc_arg_ignore(btf, &args[i]))
                        continue;

                if (btf_type_is_scalar(t)) {
                        if (reg->type != SCALAR_VALUE) {
                                verbose(env, "R%d is not a scalar\n", regno);
                                return -EINVAL;
                        }

                        if (is_kfunc_arg_constant(meta->btf, &args[i])) {
                                if (meta->arg_constant.found) {
                                        verbose(env, "verifier internal error: only one constant argument permitted\n");
                                        return -EFAULT;
                                }
                                if (!tnum_is_const(reg->var_off)) {
                                        verbose(env, "R%d must be a known constant\n", regno);
                                        return -EINVAL;
                                }
                                ret = mark_chain_precision(env, regno);
                                if (ret < 0)
                                        return ret;
                                meta->arg_constant.found = true;
                                meta->arg_constant.value = reg->var_off.value;
                        } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) {
                                meta->r0_rdonly = true;
                                is_ret_buf_sz = true;
                        } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) {
                                is_ret_buf_sz = true;
                        }

                        if (is_ret_buf_sz) {
                                if (meta->r0_size) {
                                        verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc");
                                        return -EINVAL;
                                }

                                if (!tnum_is_const(reg->var_off)) {
                                        verbose(env, "R%d is not a const\n", regno);
                                        return -EINVAL;
                                }

                                meta->r0_size = reg->var_off.value;
                                ret = mark_chain_precision(env, regno);
                                if (ret)
                                        return ret;
                        }
                        continue;
                }

                if (!btf_type_is_ptr(t)) {
                        verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t));
                        return -EINVAL;
                }

                if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) &&
                    (register_is_null(reg) || type_may_be_null(reg->type)) &&
                        !is_kfunc_arg_nullable(meta->btf, &args[i])) {
                        verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);
                        return -EACCES;
                }

                if (reg->ref_obj_id) {
                        if (is_kfunc_release(meta) && meta->ref_obj_id) {
                                verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
                                        regno, reg->ref_obj_id,
                                        meta->ref_obj_id);
                                return -EFAULT;
                        }
                        meta->ref_obj_id = reg->ref_obj_id;
                        if (is_kfunc_release(meta))
                                meta->release_regno = regno;
                }

                ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
                ref_tname = btf_name_by_offset(btf, ref_t->name_off);

                kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs);
                if (kf_arg_type < 0)
                        return kf_arg_type;

                switch (kf_arg_type) {
                case KF_ARG_PTR_TO_NULL:
                        continue;
                case KF_ARG_PTR_TO_MAP:
                        if (!reg->map_ptr) {
                                verbose(env, "pointer in R%d isn't map pointer\n", regno);
                                return -EINVAL;
                        }
                        if (meta->map.ptr && reg->map_ptr->record->wq_off >= 0) {
                                /* Use map_uid (which is unique id of inner map) to reject:
                                 * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
                                 * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
                                 * if (inner_map1 && inner_map2) {
                                 *     wq = bpf_map_lookup_elem(inner_map1);
                                 *     if (wq)
                                 *         // mismatch would have been allowed
                                 *         bpf_wq_init(wq, inner_map2);
                                 * }
                                 *
                                 * Comparing map_ptr is enough to distinguish normal and outer maps.
                                 */
                                if (meta->map.ptr != reg->map_ptr ||
                                    meta->map.uid != reg->map_uid) {
                                        verbose(env,
                                                "workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
                                                meta->map.uid, reg->map_uid);
                                        return -EINVAL;
                                }
                        }
                        meta->map.ptr = reg->map_ptr;
                        meta->map.uid = reg->map_uid;
                        fallthrough;
                case KF_ARG_PTR_TO_ALLOC_BTF_ID:
                case KF_ARG_PTR_TO_BTF_ID:
                        if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta))
                                break;

                        if (!is_trusted_reg(reg)) {
                                if (!is_kfunc_rcu(meta)) {
                                        verbose(env, "R%d must be referenced or trusted\n", regno);
                                        return -EINVAL;
                                }
                                if (!is_rcu_reg(reg)) {
                                        verbose(env, "R%d must be a rcu pointer\n", regno);
                                        return -EINVAL;
                                }
                        }
                        fallthrough;
                case KF_ARG_PTR_TO_CTX:
                case KF_ARG_PTR_TO_DYNPTR:
                case KF_ARG_PTR_TO_ITER:
                case KF_ARG_PTR_TO_LIST_HEAD:
                case KF_ARG_PTR_TO_LIST_NODE:
                case KF_ARG_PTR_TO_RB_ROOT:
                case KF_ARG_PTR_TO_RB_NODE:
                case KF_ARG_PTR_TO_MEM:
                case KF_ARG_PTR_TO_MEM_SIZE:
                case KF_ARG_PTR_TO_CALLBACK:
                case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
                case KF_ARG_PTR_TO_CONST_STR:
                case KF_ARG_PTR_TO_WORKQUEUE:
                case KF_ARG_PTR_TO_IRQ_FLAG:
                        break;
                default:
                        WARN_ON_ONCE(1);
                        return -EFAULT;
                }

                if (is_kfunc_release(meta) && reg->ref_obj_id)
                        arg_type |= OBJ_RELEASE;
                ret = check_func_arg_reg_off(env, reg, regno, arg_type);
                if (ret < 0)
                        return ret;

                switch (kf_arg_type) {
                case KF_ARG_PTR_TO_CTX:
                        if (reg->type != PTR_TO_CTX) {
                                verbose(env, "arg#%d expected pointer to ctx, but got %s\n",
                                        i, reg_type_str(env, reg->type));
                                return -EINVAL;
                        }

                        if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
                                ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));
                                if (ret < 0)
                                        return -EINVAL;
                                meta->ret_btf_id  = ret;
                        }
                        break;
                case KF_ARG_PTR_TO_ALLOC_BTF_ID:
                        if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) {
                                if (meta->func_id != special_kfunc_list[KF_bpf_obj_drop_impl]) {
                                        verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i);
                                        return -EINVAL;
                                }
                        } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) {
                                if (meta->func_id != special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) {
                                        verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i);
                                        return -EINVAL;
                                }
                        } else {
                                verbose(env, "arg#%d expected pointer to allocated object\n", i);
                                return -EINVAL;
                        }
                        if (!reg->ref_obj_id) {
                                verbose(env, "allocated object must be referenced\n");
                                return -EINVAL;
                        }
                        if (meta->btf == btf_vmlinux) {
                                meta->arg_btf = reg->btf;
                                meta->arg_btf_id = reg->btf_id;
                        }
                        break;
                case KF_ARG_PTR_TO_DYNPTR:
                {
                        enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
                        int clone_ref_obj_id = 0;

                        if (reg->type == CONST_PTR_TO_DYNPTR)
                                dynptr_arg_type |= MEM_RDONLY;

                        if (is_kfunc_arg_uninit(btf, &args[i]))
                                dynptr_arg_type |= MEM_UNINIT;

                        if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
                                dynptr_arg_type |= DYNPTR_TYPE_SKB;
                        } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) {
                                dynptr_arg_type |= DYNPTR_TYPE_XDP;
                        } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] &&
                                   (dynptr_arg_type & MEM_UNINIT)) {
                                enum bpf_dynptr_type parent_type = meta->initialized_dynptr.type;

                                if (parent_type == BPF_DYNPTR_TYPE_INVALID) {
                                        verbose(env, "verifier internal error: no dynptr type for parent of clone\n");
                                        return -EFAULT;
                                }

                                dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type);
                                clone_ref_obj_id = meta->initialized_dynptr.ref_obj_id;
                                if (dynptr_type_refcounted(parent_type) && !clone_ref_obj_id) {
                                        verbose(env, "verifier internal error: missing ref obj id for parent of clone\n");
                                        return -EFAULT;
                                }
                        }

                        ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id);
                        if (ret < 0)
                                return ret;

                        if (!(dynptr_arg_type & MEM_UNINIT)) {
                                int id = dynptr_id(env, reg);

                                if (id < 0) {
                                        verbose(env, "verifier internal error: failed to obtain dynptr id\n");
                                        return id;
                                }
                                meta->initialized_dynptr.id = id;
                                meta->initialized_dynptr.type = dynptr_get_type(env, reg);
                                meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg);
                        }

                        break;
                }
                case KF_ARG_PTR_TO_ITER:
                        if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) {
                                if (!check_css_task_iter_allowlist(env)) {
                                        verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n");
                                        return -EINVAL;
                                }
                        }
                        ret = process_iter_arg(env, regno, insn_idx, meta);
                        if (ret < 0)
                                return ret;
                        break;
                case KF_ARG_PTR_TO_LIST_HEAD:
                        if (reg->type != PTR_TO_MAP_VALUE &&
                            reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
                                verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
                                return -EINVAL;
                        }
                        if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
                                verbose(env, "allocated object must be referenced\n");
                                return -EINVAL;
                        }
                        ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta);
                        if (ret < 0)
                                return ret;
                        break;
                case KF_ARG_PTR_TO_RB_ROOT:
                        if (reg->type != PTR_TO_MAP_VALUE &&
                            reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
                                verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
                                return -EINVAL;
                        }
                        if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
                                verbose(env, "allocated object must be referenced\n");
                                return -EINVAL;
                        }
                        ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta);
                        if (ret < 0)
                                return ret;
                        break;
                case KF_ARG_PTR_TO_LIST_NODE:
                        if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
                                verbose(env, "arg#%d expected pointer to allocated object\n", i);
                                return -EINVAL;
                        }
                        if (!reg->ref_obj_id) {
                                verbose(env, "allocated object must be referenced\n");
                                return -EINVAL;
                        }
                        ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta);
                        if (ret < 0)
                                return ret;
                        break;
                case KF_ARG_PTR_TO_RB_NODE:
                        if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_remove]) {
                                if (!type_is_non_owning_ref(reg->type) || reg->ref_obj_id) {
                                        verbose(env, "rbtree_remove node input must be non-owning ref\n");
                                        return -EINVAL;
                                }
                                if (in_rbtree_lock_required_cb(env)) {
                                        verbose(env, "rbtree_remove not allowed in rbtree cb\n");
                                        return -EINVAL;
                                }
                        } else {
                                if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
                                        verbose(env, "arg#%d expected pointer to allocated object\n", i);
                                        return -EINVAL;
                                }
                                if (!reg->ref_obj_id) {
                                        verbose(env, "allocated object must be referenced\n");
                                        return -EINVAL;
                                }
                        }

                        ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta);
                        if (ret < 0)
                                return ret;
                        break;
                case KF_ARG_PTR_TO_MAP:
                        /* If argument has '__map' suffix expect 'struct bpf_map *' */
                        ref_id = *reg2btf_ids[CONST_PTR_TO_MAP];
                        ref_t = btf_type_by_id(btf_vmlinux, ref_id);
                        ref_tname = btf_name_by_offset(btf, ref_t->name_off);
                        fallthrough;
                case KF_ARG_PTR_TO_BTF_ID:
                        /* Only base_type is checked, further checks are done here */
                        if ((base_type(reg->type) != PTR_TO_BTF_ID ||
                             (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&
                            !reg2btf_ids[base_type(reg->type)]) {
                                verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type));
                                verbose(env, "expected %s or socket\n",
                                        reg_type_str(env, base_type(reg->type) |
                                                          (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));
                                return -EINVAL;
                        }
                        ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);
                        if (ret < 0)
                                return ret;
                        break;
                case KF_ARG_PTR_TO_MEM:
                        resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
                        if (IS_ERR(resolve_ret)) {
                                verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
                                        i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret));
                                return -EINVAL;
                        }
                        ret = check_mem_reg(env, reg, regno, type_size);
                        if (ret < 0)
                                return ret;
                        break;
                case KF_ARG_PTR_TO_MEM_SIZE:
                {
                        struct bpf_reg_state *buff_reg = &regs[regno];
                        const struct btf_param *buff_arg = &args[i];
                        struct bpf_reg_state *size_reg = &regs[regno + 1];
                        const struct btf_param *size_arg = &args[i + 1];

                        if (!register_is_null(buff_reg) || !is_kfunc_arg_optional(meta->btf, buff_arg)) {
                                ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1);
                                if (ret < 0) {
                                        verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1);
                                        return ret;
                                }
                        }

                        if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) {
                                if (meta->arg_constant.found) {
                                        verbose(env, "verifier internal error: only one constant argument permitted\n");
                                        return -EFAULT;
                                }
                                if (!tnum_is_const(size_reg->var_off)) {
                                        verbose(env, "R%d must be a known constant\n", regno + 1);
                                        return -EINVAL;
                                }
                                meta->arg_constant.found = true;
                                meta->arg_constant.value = size_reg->var_off.value;
                        }

                        /* Skip next '__sz' or '__szk' argument */
                        i++;
                        break;
                }
                case KF_ARG_PTR_TO_CALLBACK:
                        if (reg->type != PTR_TO_FUNC) {
                                verbose(env, "arg%d expected pointer to func\n", i);
                                return -EINVAL;
                        }
                        meta->subprogno = reg->subprogno;
                        break;
                case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
                        if (!type_is_ptr_alloc_obj(reg->type)) {
                                verbose(env, "arg#%d is neither owning or non-owning ref\n", i);
                                return -EINVAL;
                        }
                        if (!type_is_non_owning_ref(reg->type))
                                meta->arg_owning_ref = true;

                        rec = reg_btf_record(reg);
                        if (!rec) {
                                verbose(env, "verifier internal error: Couldn't find btf_record\n");
                                return -EFAULT;
                        }

                        if (rec->refcount_off < 0) {
                                verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i);
                                return -EINVAL;
                        }

                        meta->arg_btf = reg->btf;
                        meta->arg_btf_id = reg->btf_id;
                        break;
                case KF_ARG_PTR_TO_CONST_STR:
                        if (reg->type != PTR_TO_MAP_VALUE) {
                                verbose(env, "arg#%d doesn't point to a const string\n", i);
                                return -EINVAL;
                        }
                        ret = check_reg_const_str(env, reg, regno);
                        if (ret)
                                return ret;
                        break;
                case KF_ARG_PTR_TO_WORKQUEUE:
                        if (reg->type != PTR_TO_MAP_VALUE) {
                                verbose(env, "arg#%d doesn't point to a map value\n", i);
                                return -EINVAL;
                        }
                        ret = process_wq_func(env, regno, meta);
                        if (ret < 0)
                                return ret;
                        break;
                case KF_ARG_PTR_TO_IRQ_FLAG:
                        if (reg->type != PTR_TO_STACK) {
                                verbose(env, "arg#%d doesn't point to an irq flag on stack\n", i);
                                return -EINVAL;
                        }
                        ret = process_irq_flag(env, regno, meta);
                        if (ret < 0)
                                return ret;
                        break;
                }
        }

        if (is_kfunc_release(meta) && !meta->release_regno) {
                verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n",
                        func_name);
                return -EINVAL;
        }

        return 0;
}

static int fetch_kfunc_meta(struct bpf_verifier_env *env,
                            struct bpf_insn *insn,
                            struct bpf_kfunc_call_arg_meta *meta,
                            const char **kfunc_name)
{
        const struct btf_type *func, *func_proto;
        u32 func_id, *kfunc_flags;
        const char *func_name;
        struct btf *desc_btf;

        if (kfunc_name)
                *kfunc_name = NULL;

        if (!insn->imm)
                return -EINVAL;

        desc_btf = find_kfunc_desc_btf(env, insn->off);
        if (IS_ERR(desc_btf))
                return PTR_ERR(desc_btf);

        func_id = insn->imm;
        func = btf_type_by_id(desc_btf, func_id);
        func_name = btf_name_by_offset(desc_btf, func->name_off);
        if (kfunc_name)
                *kfunc_name = func_name;
        func_proto = btf_type_by_id(desc_btf, func->type);

        kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog);
        if (!kfunc_flags) {
                return -EACCES;
        }

        memset(meta, 0, sizeof(*meta));
        meta->btf = desc_btf;
        meta->func_id = func_id;
        meta->kfunc_flags = *kfunc_flags;
        meta->func_proto = func_proto;
        meta->func_name = func_name;

        return 0;
}

static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);

static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                            int *insn_idx_p)
{
        bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable;
        u32 i, nargs, ptr_type_id, release_ref_obj_id;
        struct bpf_reg_state *regs = cur_regs(env);
        const char *func_name, *ptr_type_name;
        const struct btf_type *t, *ptr_type;
        struct bpf_kfunc_call_arg_meta meta;
        struct bpf_insn_aux_data *insn_aux;
        int err, insn_idx = *insn_idx_p;
        const struct btf_param *args;
        const struct btf_type *ret_t;
        struct btf *desc_btf;

        /* skip for now, but return error when we find this in fixup_kfunc_call */
        if (!insn->imm)
                return 0;

        err = fetch_kfunc_meta(env, insn, &meta, &func_name);
        if (err == -EACCES && func_name)
                verbose(env, "calling kernel function %s is not allowed\n", func_name);
        if (err)
                return err;
        desc_btf = meta.btf;
        insn_aux = &env->insn_aux_data[insn_idx];

        insn_aux->is_iter_next = is_iter_next_kfunc(&meta);

        if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) {
                verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n");
                return -EACCES;
        }

        sleepable = is_kfunc_sleepable(&meta);
        if (sleepable && !in_sleepable(env)) {
                verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name);
                return -EACCES;
        }

        /* Check the arguments */
        err = check_kfunc_args(env, &meta, insn_idx);
        if (err < 0)
                return err;

        if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
                err = push_callback_call(env, insn, insn_idx, meta.subprogno,
                                         set_rbtree_add_callback_state);
                if (err) {
                        verbose(env, "kfunc %s#%d failed callback verification\n",
                                func_name, meta.func_id);
                        return err;
                }
        }

        if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) {
                meta.r0_size = sizeof(u64);
                meta.r0_rdonly = false;
        }

        if (is_bpf_wq_set_callback_impl_kfunc(meta.func_id)) {
                err = push_callback_call(env, insn, insn_idx, meta.subprogno,
                                         set_timer_callback_state);
                if (err) {
                        verbose(env, "kfunc %s#%d failed callback verification\n",
                                func_name, meta.func_id);
                        return err;
                }
        }

        rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta);
        rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta);

        preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
        preempt_enable = is_kfunc_bpf_preempt_enable(&meta);

        if (env->cur_state->active_rcu_lock) {
                struct bpf_func_state *state;
                struct bpf_reg_state *reg;
                u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);

                if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
                        verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
                        return -EACCES;
                }

                if (rcu_lock) {
                        verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
                        return -EINVAL;
                } else if (rcu_unlock) {
                        bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
                                if (reg->type & MEM_RCU) {
                                        reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
                                        reg->type |= PTR_UNTRUSTED;
                                }
                        }));
                        env->cur_state->active_rcu_lock = false;
                } else if (sleepable) {
                        verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
                        return -EACCES;
                }
        } else if (rcu_lock) {
                env->cur_state->active_rcu_lock = true;
        } else if (rcu_unlock) {
                verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
                return -EINVAL;
        }

        if (env->cur_state->active_preempt_locks) {
                if (preempt_disable) {
                        env->cur_state->active_preempt_locks++;
                } else if (preempt_enable) {
                        env->cur_state->active_preempt_locks--;
                } else if (sleepable) {
                        verbose(env, "kernel func %s is sleepable within non-preemptible region\n", func_name);
                        return -EACCES;
                }
        } else if (preempt_disable) {
                env->cur_state->active_preempt_locks++;
        } else if (preempt_enable) {
                verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name);
                return -EINVAL;
        }

        if (env->cur_state->active_irq_id && sleepable) {
                verbose(env, "kernel func %s is sleepable within IRQ-disabled region\n", func_name);
                return -EACCES;
        }

        /* In case of release function, we get register number of refcounted
         * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now.
         */
        if (meta.release_regno) {
                err = release_reference(env, regs[meta.release_regno].ref_obj_id);
                if (err) {
                        verbose(env, "kfunc %s#%d reference has not been acquired before\n",
                                func_name, meta.func_id);
                        return err;
                }
        }

        if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
            meta.func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
            meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
                release_ref_obj_id = regs[BPF_REG_2].ref_obj_id;
                insn_aux->insert_off = regs[BPF_REG_2].off;
                insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
                err = ref_convert_owning_non_owning(env, release_ref_obj_id);
                if (err) {
                        verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n",
                                func_name, meta.func_id);
                        return err;
                }

                err = release_reference(env, release_ref_obj_id);
                if (err) {
                        verbose(env, "kfunc %s#%d reference has not been acquired before\n",
                                func_name, meta.func_id);
                        return err;
                }
        }

        if (meta.func_id == special_kfunc_list[KF_bpf_throw]) {
                if (!bpf_jit_supports_exceptions()) {
                        verbose(env, "JIT does not support calling kfunc %s#%d\n",
                                func_name, meta.func_id);
                        return -ENOTSUPP;
                }
                env->seen_exception = true;

                /* In the case of the default callback, the cookie value passed
                 * to bpf_throw becomes the return value of the program.
                 */
                if (!env->exception_callback_subprog) {
                        err = check_return_code(env, BPF_REG_1, "R1");
                        if (err < 0)
                                return err;
                }
        }

        for (i = 0; i < CALLER_SAVED_REGS; i++)
                mark_reg_not_init(env, regs, caller_saved[i]);

        /* Check return type */
        t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL);

        if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) {
                /* Only exception is bpf_obj_new_impl */
                if (meta.btf != btf_vmlinux ||
                    (meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl] &&
                     meta.func_id != special_kfunc_list[KF_bpf_percpu_obj_new_impl] &&
                     meta.func_id != special_kfunc_list[KF_bpf_refcount_acquire_impl])) {
                        verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");
                        return -EINVAL;
                }
        }

        if (btf_type_is_scalar(t)) {
                mark_reg_unknown(env, regs, BPF_REG_0);
                mark_btf_func_reg_size(env, BPF_REG_0, t->size);
        } else if (btf_type_is_ptr(t)) {
                ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id);

                if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {
                        if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] ||
                            meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
                                struct btf_struct_meta *struct_meta;
                                struct btf *ret_btf;
                                u32 ret_btf_id;

                                if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] && !bpf_global_ma_set)
                                        return -ENOMEM;

                                if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) {
                                        verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");
                                        return -EINVAL;
                                }

                                ret_btf = env->prog->aux->btf;
                                ret_btf_id = meta.arg_constant.value;

                                /* This may be NULL due to user not supplying a BTF */
                                if (!ret_btf) {
                                        verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n");
                                        return -EINVAL;
                                }

                                ret_t = btf_type_by_id(ret_btf, ret_btf_id);
                                if (!ret_t || !__btf_type_is_struct(ret_t)) {
                                        verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n");
                                        return -EINVAL;
                                }

                                if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
                                        if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) {
                                                verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n",
                                                        ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE);
                                                return -EINVAL;
                                        }

                                        if (!bpf_global_percpu_ma_set) {
                                                mutex_lock(&bpf_percpu_ma_lock);
                                                if (!bpf_global_percpu_ma_set) {
                                                        /* Charge memory allocated with bpf_global_percpu_ma to
                                                         * root memcg. The obj_cgroup for root memcg is NULL.
                                                         */
                                                        err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL);
                                                        if (!err)
                                                                bpf_global_percpu_ma_set = true;
                                                }
                                                mutex_unlock(&bpf_percpu_ma_lock);
                                                if (err)
                                                        return err;
                                        }

                                        mutex_lock(&bpf_percpu_ma_lock);
                                        err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size);
                                        mutex_unlock(&bpf_percpu_ma_lock);
                                        if (err)
                                                return err;
                                }

                                struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id);
                                if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
                                        if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {
                                                verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");
                                                return -EINVAL;
                                        }

                                        if (struct_meta) {
                                                verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n");
                                                return -EINVAL;
                                        }
                                }

                                mark_reg_known_zero(env, regs, BPF_REG_0);
                                regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
                                regs[BPF_REG_0].btf = ret_btf;
                                regs[BPF_REG_0].btf_id = ret_btf_id;
                                if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl])
                                        regs[BPF_REG_0].type |= MEM_PERCPU;

                                insn_aux->obj_new_size = ret_t->size;
                                insn_aux->kptr_struct_meta = struct_meta;
                        } else if (meta.func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
                                mark_reg_known_zero(env, regs, BPF_REG_0);
                                regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
                                regs[BPF_REG_0].btf = meta.arg_btf;
                                regs[BPF_REG_0].btf_id = meta.arg_btf_id;

                                insn_aux->kptr_struct_meta =
                                        btf_find_struct_meta(meta.arg_btf,
                                                             meta.arg_btf_id);
                        } else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] ||
                                   meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) {
                                struct btf_field *field = meta.arg_list_head.field;

                                mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
                        } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
                                   meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {
                                struct btf_field *field = meta.arg_rbtree_root.field;

                                mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
                        } else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
                                mark_reg_known_zero(env, regs, BPF_REG_0);
                                regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED;
                                regs[BPF_REG_0].btf = desc_btf;
                                regs[BPF_REG_0].btf_id = meta.ret_btf_id;
                        } else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
                                ret_t = btf_type_by_id(desc_btf, meta.arg_constant.value);
                                if (!ret_t || !btf_type_is_struct(ret_t)) {
                                        verbose(env,
                                                "kfunc bpf_rdonly_cast type ID argument must be of a struct\n");
                                        return -EINVAL;
                                }

                                mark_reg_known_zero(env, regs, BPF_REG_0);
                                regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
                                regs[BPF_REG_0].btf = desc_btf;
                                regs[BPF_REG_0].btf_id = meta.arg_constant.value;
                        } else if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice] ||
                                   meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) {
                                enum bpf_type_flag type_flag = get_dynptr_type_flag(meta.initialized_dynptr.type);

                                mark_reg_known_zero(env, regs, BPF_REG_0);

                                if (!meta.arg_constant.found) {
                                        verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n");
                                        return -EFAULT;
                                }

                                regs[BPF_REG_0].mem_size = meta.arg_constant.value;

                                /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */
                                regs[BPF_REG_0].type = PTR_TO_MEM | type_flag;

                                if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice]) {
                                        regs[BPF_REG_0].type |= MEM_RDONLY;
                                } else {
                                        /* this will set env->seen_direct_write to true */
                                        if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {
                                                verbose(env, "the prog does not allow writes to packet data\n");
                                                return -EINVAL;
                                        }
                                }

                                if (!meta.initialized_dynptr.id) {
                                        verbose(env, "verifier internal error: no dynptr id\n");
                                        return -EFAULT;
                                }
                                regs[BPF_REG_0].dynptr_id = meta.initialized_dynptr.id;

                                /* we don't need to set BPF_REG_0's ref obj id
                                 * because packet slices are not refcounted (see
                                 * dynptr_type_refcounted)
                                 */
                        } else {
                                verbose(env, "kernel function %s unhandled dynamic return type\n",
                                        meta.func_name);
                                return -EFAULT;
                        }
                } else if (btf_type_is_void(ptr_type)) {
                        /* kfunc returning 'void *' is equivalent to returning scalar */
                        mark_reg_unknown(env, regs, BPF_REG_0);
                } else if (!__btf_type_is_struct(ptr_type)) {
                        if (!meta.r0_size) {
                                __u32 sz;

                                if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) {
                                        meta.r0_size = sz;
                                        meta.r0_rdonly = true;
                                }
                        }
                        if (!meta.r0_size) {
                                ptr_type_name = btf_name_by_offset(desc_btf,
                                                                   ptr_type->name_off);
                                verbose(env,
                                        "kernel function %s returns pointer type %s %s is not supported\n",
                                        func_name,
                                        btf_type_str(ptr_type),
                                        ptr_type_name);
                                return -EINVAL;
                        }

                        mark_reg_known_zero(env, regs, BPF_REG_0);
                        regs[BPF_REG_0].type = PTR_TO_MEM;
                        regs[BPF_REG_0].mem_size = meta.r0_size;

                        if (meta.r0_rdonly)
                                regs[BPF_REG_0].type |= MEM_RDONLY;

                        /* Ensures we don't access the memory after a release_reference() */
                        if (meta.ref_obj_id)
                                regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
                } else {
                        mark_reg_known_zero(env, regs, BPF_REG_0);
                        regs[BPF_REG_0].btf = desc_btf;
                        regs[BPF_REG_0].type = PTR_TO_BTF_ID;
                        regs[BPF_REG_0].btf_id = ptr_type_id;

                        if (meta.func_id == special_kfunc_list[KF_bpf_get_kmem_cache])
                                regs[BPF_REG_0].type |= PTR_UNTRUSTED;

                        if (is_iter_next_kfunc(&meta)) {
                                struct bpf_reg_state *cur_iter;

                                cur_iter = get_iter_from_state(env->cur_state, &meta);

                                if (cur_iter->type & MEM_RCU) /* KF_RCU_PROTECTED */
                                        regs[BPF_REG_0].type |= MEM_RCU;
                                else
                                        regs[BPF_REG_0].type |= PTR_TRUSTED;
                        }
                }

                if (is_kfunc_ret_null(&meta)) {
                        regs[BPF_REG_0].type |= PTR_MAYBE_NULL;
                        /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */
                        regs[BPF_REG_0].id = ++env->id_gen;
                }
                mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
                if (is_kfunc_acquire(&meta)) {
                        int id = acquire_reference(env, insn_idx);

                        if (id < 0)
                                return id;
                        if (is_kfunc_ret_null(&meta))
                                regs[BPF_REG_0].id = id;
                        regs[BPF_REG_0].ref_obj_id = id;
                } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {
                        ref_set_non_owning(env, &regs[BPF_REG_0]);
                }

                if (reg_may_point_to_spin_lock(&regs[BPF_REG_0]) && !regs[BPF_REG_0].id)
                        regs[BPF_REG_0].id = ++env->id_gen;
        } else if (btf_type_is_void(t)) {
                if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {
                        if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl] ||
                            meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) {
                                insn_aux->kptr_struct_meta =
                                        btf_find_struct_meta(meta.arg_btf,
                                                             meta.arg_btf_id);
                        }
                }
        }

        nargs = btf_type_vlen(meta.func_proto);
        args = (const struct btf_param *)(meta.func_proto + 1);
        for (i = 0; i < nargs; i++) {
                u32 regno = i + 1;

                t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL);
                if (btf_type_is_ptr(t))
                        mark_btf_func_reg_size(env, regno, sizeof(void *));
                else
                        /* scalar. ensured by btf_check_kfunc_arg_match() */
                        mark_btf_func_reg_size(env, regno, t->size);
        }

        if (is_iter_next_kfunc(&meta)) {
                err = process_iter_next_call(env, insn_idx, &meta);
                if (err)
                        return err;
        }

        return 0;
}

static bool check_reg_sane_offset(struct bpf_verifier_env *env,
                                  const struct bpf_reg_state *reg,
                                  enum bpf_reg_type type)
{
        bool known = tnum_is_const(reg->var_off);
        s64 val = reg->var_off.value;
        s64 smin = reg->smin_value;

        if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
                verbose(env, "math between %s pointer and %lld is not allowed\n",
                        reg_type_str(env, type), val);
                return false;
        }

        if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
                verbose(env, "%s pointer offset %d is not allowed\n",
                        reg_type_str(env, type), reg->off);
                return false;
        }

        if (smin == S64_MIN) {
                verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
                        reg_type_str(env, type));
                return false;
        }

        if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
                verbose(env, "value %lld makes %s pointer be out of bounds\n",
                        smin, reg_type_str(env, type));
                return false;
        }

        return true;
}

enum {
        REASON_BOUNDS        = -1,
        REASON_TYPE        = -2,
        REASON_PATHS        = -3,
        REASON_LIMIT        = -4,
        REASON_STACK        = -5,
};

static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
                              u32 *alu_limit, bool mask_to_left)
{
        u32 max = 0, ptr_limit = 0;

        switch (ptr_reg->type) {
        case PTR_TO_STACK:
                /* Offset 0 is out-of-bounds, but acceptable start for the
                 * left direction, see BPF_REG_FP. Also, unknown scalar
                 * offset where we would need to deal with min/max bounds is
                 * currently prohibited for unprivileged.
                 */
                max = MAX_BPF_STACK + mask_to_left;
                ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off);
                break;
        case PTR_TO_MAP_VALUE:
                max = ptr_reg->map_ptr->value_size;
                ptr_limit = (mask_to_left ?
                             ptr_reg->smin_value :
                             ptr_reg->umax_value) + ptr_reg->off;
                break;
        default:
                return REASON_TYPE;
        }

        if (ptr_limit >= max)
                return REASON_LIMIT;
        *alu_limit = ptr_limit;
        return 0;
}

static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
                                    const struct bpf_insn *insn)
{
        return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;
}

static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
                                       u32 alu_state, u32 alu_limit)
{
        /* If we arrived here from different branches with different
         * state or limits to sanitize, then this won't work.
         */
        if (aux->alu_state &&
            (aux->alu_state != alu_state ||
             aux->alu_limit != alu_limit))
                return REASON_PATHS;

        /* Corresponding fixup done in do_misc_fixups(). */
        aux->alu_state = alu_state;
        aux->alu_limit = alu_limit;
        return 0;
}

static int sanitize_val_alu(struct bpf_verifier_env *env,
                            struct bpf_insn *insn)
{
        struct bpf_insn_aux_data *aux = cur_aux(env);

        if (can_skip_alu_sanitation(env, insn))
                return 0;

        return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
}

static bool sanitize_needed(u8 opcode)
{
        return opcode == BPF_ADD || opcode == BPF_SUB;
}

struct bpf_sanitize_info {
        struct bpf_insn_aux_data aux;
        bool mask_to_left;
};

static struct bpf_verifier_state *
sanitize_speculative_path(struct bpf_verifier_env *env,
                          const struct bpf_insn *insn,
                          u32 next_idx, u32 curr_idx)
{
        struct bpf_verifier_state *branch;
        struct bpf_reg_state *regs;

        branch = push_stack(env, next_idx, curr_idx, true);
        if (branch && insn) {
                regs = branch->frame[branch->curframe]->regs;
                if (BPF_SRC(insn->code) == BPF_K) {
                        mark_reg_unknown(env, regs, insn->dst_reg);
                } else if (BPF_SRC(insn->code) == BPF_X) {
                        mark_reg_unknown(env, regs, insn->dst_reg);
                        mark_reg_unknown(env, regs, insn->src_reg);
                }
        }
        return branch;
}

static int sanitize_ptr_alu(struct bpf_verifier_env *env,
                            struct bpf_insn *insn,
                            const struct bpf_reg_state *ptr_reg,
                            const struct bpf_reg_state *off_reg,
                            struct bpf_reg_state *dst_reg,
                            struct bpf_sanitize_info *info,
                            const bool commit_window)
{
        struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
        struct bpf_verifier_state *vstate = env->cur_state;
        bool off_is_imm = tnum_is_const(off_reg->var_off);
        bool off_is_neg = off_reg->smin_value < 0;
        bool ptr_is_dst_reg = ptr_reg == dst_reg;
        u8 opcode = BPF_OP(insn->code);
        u32 alu_state, alu_limit;
        struct bpf_reg_state tmp;
        bool ret;
        int err;

        if (can_skip_alu_sanitation(env, insn))
                return 0;

        /* We already marked aux for masking from non-speculative
         * paths, thus we got here in the first place. We only care
         * to explore bad access from here.
         */
        if (vstate->speculative)
                goto do_sim;

        if (!commit_window) {
                if (!tnum_is_const(off_reg->var_off) &&
                    (off_reg->smin_value < 0) != (off_reg->smax_value < 0))
                        return REASON_BOUNDS;

                info->mask_to_left = (opcode == BPF_ADD &&  off_is_neg) ||
                                     (opcode == BPF_SUB && !off_is_neg);
        }

        err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left);
        if (err < 0)
                return err;

        if (commit_window) {
                /* In commit phase we narrow the masking window based on
                 * the observed pointer move after the simulated operation.
                 */
                alu_state = info->aux.alu_state;
                alu_limit = abs(info->aux.alu_limit - alu_limit);
        } else {
                alu_state  = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
                alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
                alu_state |= ptr_is_dst_reg ?
                             BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;

                /* Limit pruning on unknown scalars to enable deep search for
                 * potential masking differences from other program paths.
                 */
                if (!off_is_imm)
                        env->explore_alu_limits = true;
        }

        err = update_alu_sanitation_state(aux, alu_state, alu_limit);
        if (err < 0)
                return err;
do_sim:
        /* If we're in commit phase, we're done here given we already
         * pushed the truncated dst_reg into the speculative verification
         * stack.
         *
         * Also, when register is a known constant, we rewrite register-based
         * operation to immediate-based, and thus do not need masking (and as
         * a consequence, do not need to simulate the zero-truncation either).
         */
        if (commit_window || off_is_imm)
                return 0;

        /* Simulate and find potential out-of-bounds access under
         * speculative execution from truncation as a result of
         * masking when off was not within expected range. If off
         * sits in dst, then we temporarily need to move ptr there
         * to simulate dst (== 0) +/-= ptr. Needed, for example,
         * for cases where we use K-based arithmetic in one direction
         * and truncated reg-based in the other in order to explore
         * bad access.
         */
        if (!ptr_is_dst_reg) {
                tmp = *dst_reg;
                copy_register_state(dst_reg, ptr_reg);
        }
        ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1,
                                        env->insn_idx);
        if (!ptr_is_dst_reg && ret)
                *dst_reg = tmp;
        return !ret ? REASON_STACK : 0;
}

static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
{
        struct bpf_verifier_state *vstate = env->cur_state;

        /* If we simulate paths under speculation, we don't update the
         * insn as 'seen' such that when we verify unreachable paths in
         * the non-speculative domain, sanitize_dead_code() can still
         * rewrite/sanitize them.
         */
        if (!vstate->speculative)
                env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
}

static int sanitize_err(struct bpf_verifier_env *env,
                        const struct bpf_insn *insn, int reason,
                        const struct bpf_reg_state *off_reg,
                        const struct bpf_reg_state *dst_reg)
{
        static const char *err = "pointer arithmetic with it prohibited for !root";
        const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
        u32 dst = insn->dst_reg, src = insn->src_reg;

        switch (reason) {
        case REASON_BOUNDS:
                verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
                        off_reg == dst_reg ? dst : src, err);
                break;
        case REASON_TYPE:
                verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
                        off_reg == dst_reg ? src : dst, err);
                break;
        case REASON_PATHS:
                verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
                        dst, op, err);
                break;
        case REASON_LIMIT:
                verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",
                        dst, op, err);
                break;
        case REASON_STACK:
                verbose(env, "R%d could not be pushed for speculative verification, %s\n",
                        dst, err);
                break;
        default:
                verbose(env, "verifier internal error: unknown reason (%d)\n",
                        reason);
                break;
        }

        return -EACCES;
}

/* check that stack access falls within stack limits and that 'reg' doesn't
 * have a variable offset.
 *
 * Variable offset is prohibited for unprivileged mode for simplicity since it
 * requires corresponding support in Spectre masking for stack ALU.  See also
 * retrieve_ptr_limit().
 *
 *
 * 'off' includes 'reg->off'.
 */
static int check_stack_access_for_ptr_arithmetic(
                                struct bpf_verifier_env *env,
                                int regno,
                                const struct bpf_reg_state *reg,
                                int off)
{
        if (!tnum_is_const(reg->var_off)) {
                char tn_buf[48];

                tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
                verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
                        regno, tn_buf, off);
                return -EACCES;
        }

        if (off >= 0 || off < -MAX_BPF_STACK) {
                verbose(env, "R%d stack pointer arithmetic goes out of range, "
                        "prohibited for !root; off=%d\n", regno, off);
                return -EACCES;
        }

        return 0;
}

static int sanitize_check_bounds(struct bpf_verifier_env *env,
                                 const struct bpf_insn *insn,
                                 const struct bpf_reg_state *dst_reg)
{
        u32 dst = insn->dst_reg;

        /* For unprivileged we require that resulting offset must be in bounds
         * in order to be able to sanitize access later on.
         */
        if (env->bypass_spec_v1)
                return 0;

        switch (dst_reg->type) {
        case PTR_TO_STACK:
                if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,
                                        dst_reg->off + dst_reg->var_off.value))
                        return -EACCES;
                break;
        case PTR_TO_MAP_VALUE:
                if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) {
                        verbose(env, "R%d pointer arithmetic of map value goes out of range, "
                                "prohibited for !root\n", dst);
                        return -EACCES;
                }
                break;
        default:
                break;
        }

        return 0;
}

/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
 * Caller should also handle BPF_MOV case separately.
 * If we return -EACCES, caller may want to try again treating pointer as a
 * scalar.  So we only emit a diagnostic if !env->allow_ptr_leaks.
 */
static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
                                   struct bpf_insn *insn,
                                   const struct bpf_reg_state *ptr_reg,
                                   const struct bpf_reg_state *off_reg)
{
        struct bpf_verifier_state *vstate = env->cur_state;
        struct bpf_func_state *state = vstate->frame[vstate->curframe];
        struct bpf_reg_state *regs = state->regs, *dst_reg;
        bool known = tnum_is_const(off_reg->var_off);
        s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
            smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
        u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
            umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
        struct bpf_sanitize_info info = {};
        u8 opcode = BPF_OP(insn->code);
        u32 dst = insn->dst_reg;
        int ret;

        dst_reg = &regs[dst];

        if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
            smin_val > smax_val || umin_val > umax_val) {
                /* Taint dst register if offset had invalid bounds derived from
                 * e.g. dead branches.
                 */
                __mark_reg_unknown(env, dst_reg);
                return 0;
        }

        if (BPF_CLASS(insn->code) != BPF_ALU64) {
                /* 32-bit ALU ops on pointers produce (meaningless) scalars */
                if (opcode == BPF_SUB && env->allow_ptr_leaks) {
                        __mark_reg_unknown(env, dst_reg);
                        return 0;
                }

                verbose(env,
                        "R%d 32-bit pointer arithmetic prohibited\n",
                        dst);
                return -EACCES;
        }

        if (ptr_reg->type & PTR_MAYBE_NULL) {
                verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
                        dst, reg_type_str(env, ptr_reg->type));
                return -EACCES;
        }

        switch (base_type(ptr_reg->type)) {
        case PTR_TO_CTX:
        case PTR_TO_MAP_VALUE:
        case PTR_TO_MAP_KEY:
        case PTR_TO_STACK:
        case PTR_TO_PACKET_META:
        case PTR_TO_PACKET:
        case PTR_TO_TP_BUFFER:
        case PTR_TO_BTF_ID:
        case PTR_TO_MEM:
        case PTR_TO_BUF:
        case PTR_TO_FUNC:
        case CONST_PTR_TO_DYNPTR:
                break;
        case PTR_TO_FLOW_KEYS:
                if (known)
                        break;
                fallthrough;
        case CONST_PTR_TO_MAP:
                /* smin_val represents the known value */
                if (known && smin_val == 0 && opcode == BPF_ADD)
                        break;
                fallthrough;
        default:
                verbose(env, "R%d pointer arithmetic on %s prohibited\n",
                        dst, reg_type_str(env, ptr_reg->type));
                return -EACCES;
        }

        /* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
         * The id may be overwritten later if we create a new variable offset.
         */
        dst_reg->type = ptr_reg->type;
        dst_reg->id = ptr_reg->id;

        if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
            !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
                return -EINVAL;

        /* pointer types do not carry 32-bit bounds at the moment. */
        __mark_reg32_unbounded(dst_reg);

        if (sanitize_needed(opcode)) {
                ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
                                       &info, false);
                if (ret < 0)
                        return sanitize_err(env, insn, ret, off_reg, dst_reg);
        }

        switch (opcode) {
        case BPF_ADD:
                /* We can take a fixed offset as long as it doesn't overflow
                 * the s32 'off' field
                 */
                if (known && (ptr_reg->off + smin_val ==
                              (s64)(s32)(ptr_reg->off + smin_val))) {
                        /* pointer += K.  Accumulate it into fixed offset */
                        dst_reg->smin_value = smin_ptr;
                        dst_reg->smax_value = smax_ptr;
                        dst_reg->umin_value = umin_ptr;
                        dst_reg->umax_value = umax_ptr;
                        dst_reg->var_off = ptr_reg->var_off;
                        dst_reg->off = ptr_reg->off + smin_val;
                        dst_reg->raw = ptr_reg->raw;
                        break;
                }
                /* A new variable offset is created.  Note that off_reg->off
                 * == 0, since it's a scalar.
                 * dst_reg gets the pointer type and since some positive
                 * integer value was added to the pointer, give it a new 'id'
                 * if it's a PTR_TO_PACKET.
                 * this creates a new 'base' pointer, off_reg (variable) gets
                 * added into the variable offset, and we copy the fixed offset
                 * from ptr_reg.
                 */
                if (check_add_overflow(smin_ptr, smin_val, &dst_reg->smin_value) ||
                    check_add_overflow(smax_ptr, smax_val, &dst_reg->smax_value)) {
                        dst_reg->smin_value = S64_MIN;
                        dst_reg->smax_value = S64_MAX;
                }
                if (check_add_overflow(umin_ptr, umin_val, &dst_reg->umin_value) ||
                    check_add_overflow(umax_ptr, umax_val, &dst_reg->umax_value)) {
                        dst_reg->umin_value = 0;
                        dst_reg->umax_value = U64_MAX;
                }
                dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
                dst_reg->off = ptr_reg->off;
                dst_reg->raw = ptr_reg->raw;
                if (reg_is_pkt_pointer(ptr_reg)) {
                        dst_reg->id = ++env->id_gen;
                        /* something was added to pkt_ptr, set range to zero */
                        memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
                }
                break;
        case BPF_SUB:
                if (dst_reg == off_reg) {
                        /* scalar -= pointer.  Creates an unknown scalar */
                        verbose(env, "R%d tried to subtract pointer from scalar\n",
                                dst);
                        return -EACCES;
                }
                /* We don't allow subtraction from FP, because (according to
                 * test_verifier.c test "invalid fp arithmetic", JITs might not
                 * be able to deal with it.
                 */
                if (ptr_reg->type == PTR_TO_STACK) {
                        verbose(env, "R%d subtraction from stack pointer prohibited\n",
                                dst);
                        return -EACCES;
                }
                if (known && (ptr_reg->off - smin_val ==
                              (s64)(s32)(ptr_reg->off - smin_val))) {
                        /* pointer -= K.  Subtract it from fixed offset */
                        dst_reg->smin_value = smin_ptr;
                        dst_reg->smax_value = smax_ptr;
                        dst_reg->umin_value = umin_ptr;
                        dst_reg->umax_value = umax_ptr;
                        dst_reg->var_off = ptr_reg->var_off;
                        dst_reg->id = ptr_reg->id;
                        dst_reg->off = ptr_reg->off - smin_val;
                        dst_reg->raw = ptr_reg->raw;
                        break;
                }
                /* A new variable offset is created.  If the subtrahend is known
                 * nonnegative, then any reg->range we had before is still good.
                 */
                if (check_sub_overflow(smin_ptr, smax_val, &dst_reg->smin_value) ||
                    check_sub_overflow(smax_ptr, smin_val, &dst_reg->smax_value)) {
                        /* Overflow possible, we know nothing */
                        dst_reg->smin_value = S64_MIN;
                        dst_reg->smax_value = S64_MAX;
                }
                if (umin_ptr < umax_val) {
                        /* Overflow possible, we know nothing */
                        dst_reg->umin_value = 0;
                        dst_reg->umax_value = U64_MAX;
                } else {
                        /* Cannot overflow (as long as bounds are consistent) */
                        dst_reg->umin_value = umin_ptr - umax_val;
                        dst_reg->umax_value = umax_ptr - umin_val;
                }
                dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
                dst_reg->off = ptr_reg->off;
                dst_reg->raw = ptr_reg->raw;
                if (reg_is_pkt_pointer(ptr_reg)) {
                        dst_reg->id = ++env->id_gen;
                        /* something was added to pkt_ptr, set range to zero */
                        if (smin_val < 0)
                                memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
                }
                break;
        case BPF_AND:
        case BPF_OR:
        case BPF_XOR:
                /* bitwise ops on pointers are troublesome, prohibit. */
                verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
                        dst, bpf_alu_string[opcode >> 4]);
                return -EACCES;
        default:
                /* other operators (e.g. MUL,LSH) produce non-pointer results */
                verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
                        dst, bpf_alu_string[opcode >> 4]);
                return -EACCES;
        }

        if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
                return -EINVAL;
        reg_bounds_sync(dst_reg);
        if (sanitize_check_bounds(env, insn, dst_reg) < 0)
                return -EACCES;
        if (sanitize_needed(opcode)) {
                ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,
                                       &info, true);
                if (ret < 0)
                        return sanitize_err(env, insn, ret, off_reg, dst_reg);
        }

        return 0;
}

static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,
                                 struct bpf_reg_state *src_reg)
{
        s32 *dst_smin = &dst_reg->s32_min_value;
        s32 *dst_smax = &dst_reg->s32_max_value;
        u32 *dst_umin = &dst_reg->u32_min_value;
        u32 *dst_umax = &dst_reg->u32_max_value;

        if (check_add_overflow(*dst_smin, src_reg->s32_min_value, dst_smin) ||
            check_add_overflow(*dst_smax, src_reg->s32_max_value, dst_smax)) {
                *dst_smin = S32_MIN;
                *dst_smax = S32_MAX;
        }
        if (check_add_overflow(*dst_umin, src_reg->u32_min_value, dst_umin) ||
            check_add_overflow(*dst_umax, src_reg->u32_max_value, dst_umax)) {
                *dst_umin = 0;
                *dst_umax = U32_MAX;
        }
}

static void scalar_min_max_add(struct bpf_reg_state *dst_reg,
                               struct bpf_reg_state *src_reg)
{
        s64 *dst_smin = &dst_reg->smin_value;
        s64 *dst_smax = &dst_reg->smax_value;
        u64 *dst_umin = &dst_reg->umin_value;
        u64 *dst_umax = &dst_reg->umax_value;

        if (check_add_overflow(*dst_smin, src_reg->smin_value, dst_smin) ||
            check_add_overflow(*dst_smax, src_reg->smax_value, dst_smax)) {
                *dst_smin = S64_MIN;
                *dst_smax = S64_MAX;
        }
        if (check_add_overflow(*dst_umin, src_reg->umin_value, dst_umin) ||
            check_add_overflow(*dst_umax, src_reg->umax_value, dst_umax)) {
                *dst_umin = 0;
                *dst_umax = U64_MAX;
        }
}

static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,
                                 struct bpf_reg_state *src_reg)
{
        s32 *dst_smin = &dst_reg->s32_min_value;
        s32 *dst_smax = &dst_reg->s32_max_value;
        u32 umin_val = src_reg->u32_min_value;
        u32 umax_val = src_reg->u32_max_value;

        if (check_sub_overflow(*dst_smin, src_reg->s32_max_value, dst_smin) ||
            check_sub_overflow(*dst_smax, src_reg->s32_min_value, dst_smax)) {
                /* Overflow possible, we know nothing */
                *dst_smin = S32_MIN;
                *dst_smax = S32_MAX;
        }
        if (dst_reg->u32_min_value < umax_val) {
                /* Overflow possible, we know nothing */
                dst_reg->u32_min_value = 0;
                dst_reg->u32_max_value = U32_MAX;
        } else {
                /* Cannot overflow (as long as bounds are consistent) */
                dst_reg->u32_min_value -= umax_val;
                dst_reg->u32_max_value -= umin_val;
        }
}

static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,
                               struct bpf_reg_state *src_reg)
{
        s64 *dst_smin = &dst_reg->smin_value;
        s64 *dst_smax = &dst_reg->smax_value;
        u64 umin_val = src_reg->umin_value;
        u64 umax_val = src_reg->umax_value;

        if (check_sub_overflow(*dst_smin, src_reg->smax_value, dst_smin) ||
            check_sub_overflow(*dst_smax, src_reg->smin_value, dst_smax)) {
                /* Overflow possible, we know nothing */
                *dst_smin = S64_MIN;
                *dst_smax = S64_MAX;
        }
        if (dst_reg->umin_value < umax_val) {
                /* Overflow possible, we know nothing */
                dst_reg->umin_value = 0;
                dst_reg->umax_value = U64_MAX;
        } else {
                /* Cannot overflow (as long as bounds are consistent) */
                dst_reg->umin_value -= umax_val;
                dst_reg->umax_value -= umin_val;
        }
}

static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,
                                 struct bpf_reg_state *src_reg)
{
        s32 *dst_smin = &dst_reg->s32_min_value;
        s32 *dst_smax = &dst_reg->s32_max_value;
        u32 *dst_umin = &dst_reg->u32_min_value;
        u32 *dst_umax = &dst_reg->u32_max_value;
        s32 tmp_prod[4];

        if (check_mul_overflow(*dst_umax, src_reg->u32_max_value, dst_umax) ||
            check_mul_overflow(*dst_umin, src_reg->u32_min_value, dst_umin)) {
                /* Overflow possible, we know nothing */
                *dst_umin = 0;
                *dst_umax = U32_MAX;
        }
        if (check_mul_overflow(*dst_smin, src_reg->s32_min_value, &tmp_prod[0]) ||
            check_mul_overflow(*dst_smin, src_reg->s32_max_value, &tmp_prod[1]) ||
            check_mul_overflow(*dst_smax, src_reg->s32_min_value, &tmp_prod[2]) ||
            check_mul_overflow(*dst_smax, src_reg->s32_max_value, &tmp_prod[3])) {
                /* Overflow possible, we know nothing */
                *dst_smin = S32_MIN;
                *dst_smax = S32_MAX;
        } else {
                *dst_smin = min_array(tmp_prod, 4);
                *dst_smax = max_array(tmp_prod, 4);
        }
}

static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
                               struct bpf_reg_state *src_reg)
{
        s64 *dst_smin = &dst_reg->smin_value;
        s64 *dst_smax = &dst_reg->smax_value;
        u64 *dst_umin = &dst_reg->umin_value;
        u64 *dst_umax = &dst_reg->umax_value;
        s64 tmp_prod[4];

        if (check_mul_overflow(*dst_umax, src_reg->umax_value, dst_umax) ||
            check_mul_overflow(*dst_umin, src_reg->umin_value, dst_umin)) {
                /* Overflow possible, we know nothing */
                *dst_umin = 0;
                *dst_umax = U64_MAX;
        }
        if (check_mul_overflow(*dst_smin, src_reg->smin_value, &tmp_prod[0]) ||
            check_mul_overflow(*dst_smin, src_reg->smax_value, &tmp_prod[1]) ||
            check_mul_overflow(*dst_smax, src_reg->smin_value, &tmp_prod[2]) ||
            check_mul_overflow(*dst_smax, src_reg->smax_value, &tmp_prod[3])) {
                /* Overflow possible, we know nothing */
                *dst_smin = S64_MIN;
                *dst_smax = S64_MAX;
        } else {
                *dst_smin = min_array(tmp_prod, 4);
                *dst_smax = max_array(tmp_prod, 4);
        }
}

static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
                                 struct bpf_reg_state *src_reg)
{
        bool src_known = tnum_subreg_is_const(src_reg->var_off);
        bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
        struct tnum var32_off = tnum_subreg(dst_reg->var_off);
        u32 umax_val = src_reg->u32_max_value;

        if (src_known && dst_known) {
                __mark_reg32_known(dst_reg, var32_off.value);
                return;
        }

        /* We get our minimum from the var_off, since that's inherently
         * bitwise.  Our maximum is the minimum of the operands' maxima.
         */
        dst_reg->u32_min_value = var32_off.value;
        dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val);

        /* Safe to set s32 bounds by casting u32 result into s32 when u32
         * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
         */
        if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
                dst_reg->s32_min_value = dst_reg->u32_min_value;
                dst_reg->s32_max_value = dst_reg->u32_max_value;
        } else {
                dst_reg->s32_min_value = S32_MIN;
                dst_reg->s32_max_value = S32_MAX;
        }
}

static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
                               struct bpf_reg_state *src_reg)
{
        bool src_known = tnum_is_const(src_reg->var_off);
        bool dst_known = tnum_is_const(dst_reg->var_off);
        u64 umax_val = src_reg->umax_value;

        if (src_known && dst_known) {
                __mark_reg_known(dst_reg, dst_reg->var_off.value);
                return;
        }

        /* We get our minimum from the var_off, since that's inherently
         * bitwise.  Our maximum is the minimum of the operands' maxima.
         */
        dst_reg->umin_value = dst_reg->var_off.value;
        dst_reg->umax_value = min(dst_reg->umax_value, umax_val);

        /* Safe to set s64 bounds by casting u64 result into s64 when u64
         * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
         */
        if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
                dst_reg->smin_value = dst_reg->umin_value;
                dst_reg->smax_value = dst_reg->umax_value;
        } else {
                dst_reg->smin_value = S64_MIN;
                dst_reg->smax_value = S64_MAX;
        }
        /* We may learn something more from the var_off */
        __update_reg_bounds(dst_reg);
}

static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
                                struct bpf_reg_state *src_reg)
{
        bool src_known = tnum_subreg_is_const(src_reg->var_off);
        bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
        struct tnum var32_off = tnum_subreg(dst_reg->var_off);
        u32 umin_val = src_reg->u32_min_value;

        if (src_known && dst_known) {
                __mark_reg32_known(dst_reg, var32_off.value);
                return;
        }

        /* We get our maximum from the var_off, and our minimum is the
         * maximum of the operands' minima
         */
        dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val);
        dst_reg->u32_max_value = var32_off.value | var32_off.mask;

        /* Safe to set s32 bounds by casting u32 result into s32 when u32
         * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
         */
        if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
                dst_reg->s32_min_value = dst_reg->u32_min_value;
                dst_reg->s32_max_value = dst_reg->u32_max_value;
        } else {
                dst_reg->s32_min_value = S32_MIN;
                dst_reg->s32_max_value = S32_MAX;
        }
}

static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
                              struct bpf_reg_state *src_reg)
{
        bool src_known = tnum_is_const(src_reg->var_off);
        bool dst_known = tnum_is_const(dst_reg->var_off);
        u64 umin_val = src_reg->umin_value;

        if (src_known && dst_known) {
                __mark_reg_known(dst_reg, dst_reg->var_off.value);
                return;
        }

        /* We get our maximum from the var_off, and our minimum is the
         * maximum of the operands' minima
         */
        dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
        dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;

        /* Safe to set s64 bounds by casting u64 result into s64 when u64
         * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
         */
        if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
                dst_reg->smin_value = dst_reg->umin_value;
                dst_reg->smax_value = dst_reg->umax_value;
        } else {
                dst_reg->smin_value = S64_MIN;
                dst_reg->smax_value = S64_MAX;
        }
        /* We may learn something more from the var_off */
        __update_reg_bounds(dst_reg);
}

static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
                                 struct bpf_reg_state *src_reg)
{
        bool src_known = tnum_subreg_is_const(src_reg->var_off);
        bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
        struct tnum var32_off = tnum_subreg(dst_reg->var_off);

        if (src_known && dst_known) {
                __mark_reg32_known(dst_reg, var32_off.value);
                return;
        }

        /* We get both minimum and maximum from the var32_off. */
        dst_reg->u32_min_value = var32_off.value;
        dst_reg->u32_max_value = var32_off.value | var32_off.mask;

        /* Safe to set s32 bounds by casting u32 result into s32 when u32
         * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
         */
        if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
                dst_reg->s32_min_value = dst_reg->u32_min_value;
                dst_reg->s32_max_value = dst_reg->u32_max_value;
        } else {
                dst_reg->s32_min_value = S32_MIN;
                dst_reg->s32_max_value = S32_MAX;
        }
}

static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
                               struct bpf_reg_state *src_reg)
{
        bool src_known = tnum_is_const(src_reg->var_off);
        bool dst_known = tnum_is_const(dst_reg->var_off);

        if (src_known && dst_known) {
                /* dst_reg->var_off.value has been updated earlier */
                __mark_reg_known(dst_reg, dst_reg->var_off.value);
                return;
        }

        /* We get both minimum and maximum from the var_off. */
        dst_reg->umin_value = dst_reg->var_off.value;
        dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;

        /* Safe to set s64 bounds by casting u64 result into s64 when u64
         * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
         */
        if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
                dst_reg->smin_value = dst_reg->umin_value;
                dst_reg->smax_value = dst_reg->umax_value;
        } else {
                dst_reg->smin_value = S64_MIN;
                dst_reg->smax_value = S64_MAX;
        }

        __update_reg_bounds(dst_reg);
}

static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
                                   u64 umin_val, u64 umax_val)
{
        /* We lose all sign bit information (except what we can pick
         * up from var_off)
         */
        dst_reg->s32_min_value = S32_MIN;
        dst_reg->s32_max_value = S32_MAX;
        /* If we might shift our top bit out, then we know nothing */
        if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) {
                dst_reg->u32_min_value = 0;
                dst_reg->u32_max_value = U32_MAX;
        } else {
                dst_reg->u32_min_value <<= umin_val;
                dst_reg->u32_max_value <<= umax_val;
        }
}

static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
                                 struct bpf_reg_state *src_reg)
{
        u32 umax_val = src_reg->u32_max_value;
        u32 umin_val = src_reg->u32_min_value;
        /* u32 alu operation will zext upper bits */
        struct tnum subreg = tnum_subreg(dst_reg->var_off);

        __scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
        dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val));
        /* Not required but being careful mark reg64 bounds as unknown so
         * that we are forced to pick them up from tnum and zext later and
         * if some path skips this step we are still safe.
         */
        __mark_reg64_unbounded(dst_reg);
        __update_reg32_bounds(dst_reg);
}

static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
                                   u64 umin_val, u64 umax_val)
{
        /* Special case <<32 because it is a common compiler pattern to sign
         * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are
         * positive we know this shift will also be positive so we can track
         * bounds correctly. Otherwise we lose all sign bit information except
         * what we can pick up from var_off. Perhaps we can generalize this
         * later to shifts of any length.
         */
        if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0)
                dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32;
        else
                dst_reg->smax_value = S64_MAX;

        if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0)
                dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32;
        else
                dst_reg->smin_value = S64_MIN;

        /* If we might shift our top bit out, then we know nothing */
        if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
                dst_reg->umin_value = 0;
                dst_reg->umax_value = U64_MAX;
        } else {
                dst_reg->umin_value <<= umin_val;
                dst_reg->umax_value <<= umax_val;
        }
}

static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,
                               struct bpf_reg_state *src_reg)
{
        u64 umax_val = src_reg->umax_value;
        u64 umin_val = src_reg->umin_value;

        /* scalar64 calc uses 32bit unshifted bounds so must be called first */
        __scalar64_min_max_lsh(dst_reg, umin_val, umax_val);
        __scalar32_min_max_lsh(dst_reg, umin_val, umax_val);

        dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
        /* We may learn something more from the var_off */
        __update_reg_bounds(dst_reg);
}

static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
                                 struct bpf_reg_state *src_reg)
{
        struct tnum subreg = tnum_subreg(dst_reg->var_off);
        u32 umax_val = src_reg->u32_max_value;
        u32 umin_val = src_reg->u32_min_value;

        /* BPF_RSH is an unsigned shift.  If the value in dst_reg might
         * be negative, then either:
         * 1) src_reg might be zero, so the sign bit of the result is
         *    unknown, so we lose our signed bounds
         * 2) it's known negative, thus the unsigned bounds capture the
         *    signed bounds
         * 3) the signed bounds cross zero, so they tell us nothing
         *    about the result
         * If the value in dst_reg is known nonnegative, then again the
         * unsigned bounds capture the signed bounds.
         * Thus, in all cases it suffices to blow away our signed bounds
         * and rely on inferring new ones from the unsigned bounds and
         * var_off of the result.
         */
        dst_reg->s32_min_value = S32_MIN;
        dst_reg->s32_max_value = S32_MAX;

        dst_reg->var_off = tnum_rshift(subreg, umin_val);
        dst_reg->u32_min_value >>= umax_val;
        dst_reg->u32_max_value >>= umin_val;

        __mark_reg64_unbounded(dst_reg);
        __update_reg32_bounds(dst_reg);
}

static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
                               struct bpf_reg_state *src_reg)
{
        u64 umax_val = src_reg->umax_value;
        u64 umin_val = src_reg->umin_value;

        /* BPF_RSH is an unsigned shift.  If the value in dst_reg might
         * be negative, then either:
         * 1) src_reg might be zero, so the sign bit of the result is
         *    unknown, so we lose our signed bounds
         * 2) it's known negative, thus the unsigned bounds capture the
         *    signed bounds
         * 3) the signed bounds cross zero, so they tell us nothing
         *    about the result
         * If the value in dst_reg is known nonnegative, then again the
         * unsigned bounds capture the signed bounds.
         * Thus, in all cases it suffices to blow away our signed bounds
         * and rely on inferring new ones from the unsigned bounds and
         * var_off of the result.
         */
        dst_reg->smin_value = S64_MIN;
        dst_reg->smax_value = S64_MAX;
        dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
        dst_reg->umin_value >>= umax_val;
        dst_reg->umax_value >>= umin_val;

        /* Its not easy to operate on alu32 bounds here because it depends
         * on bits being shifted in. Take easy way out and mark unbounded
         * so we can recalculate later from tnum.
         */
        __mark_reg32_unbounded(dst_reg);
        __update_reg_bounds(dst_reg);
}

static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
                                  struct bpf_reg_state *src_reg)
{
        u64 umin_val = src_reg->u32_min_value;

        /* Upon reaching here, src_known is true and
         * umax_val is equal to umin_val.
         */
        dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val);
        dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val);

        dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);

        /* blow away the dst_reg umin_value/umax_value and rely on
         * dst_reg var_off to refine the result.
         */
        dst_reg->u32_min_value = 0;
        dst_reg->u32_max_value = U32_MAX;

        __mark_reg64_unbounded(dst_reg);
        __update_reg32_bounds(dst_reg);
}

static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,
                                struct bpf_reg_state *src_reg)
{
        u64 umin_val = src_reg->umin_value;

        /* Upon reaching here, src_known is true and umax_val is equal
         * to umin_val.
         */
        dst_reg->smin_value >>= umin_val;
        dst_reg->smax_value >>= umin_val;

        dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);

        /* blow away the dst_reg umin_value/umax_value and rely on
         * dst_reg var_off to refine the result.
         */
        dst_reg->umin_value = 0;
        dst_reg->umax_value = U64_MAX;

        /* Its not easy to operate on alu32 bounds here because it depends
         * on bits being shifted in from upper 32-bits. Take easy way out
         * and mark unbounded so we can recalculate later from tnum.
         */
        __mark_reg32_unbounded(dst_reg);
        __update_reg_bounds(dst_reg);
}

static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn,
                                             const struct bpf_reg_state *src_reg)
{
        bool src_is_const = false;
        u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;

        if (insn_bitness == 32) {
                if (tnum_subreg_is_const(src_reg->var_off)
                    && src_reg->s32_min_value == src_reg->s32_max_value
                    && src_reg->u32_min_value == src_reg->u32_max_value)
                        src_is_const = true;
        } else {
                if (tnum_is_const(src_reg->var_off)
                    && src_reg->smin_value == src_reg->smax_value
                    && src_reg->umin_value == src_reg->umax_value)
                        src_is_const = true;
        }

        switch (BPF_OP(insn->code)) {
        case BPF_ADD:
        case BPF_SUB:
        case BPF_AND:
        case BPF_XOR:
        case BPF_OR:
        case BPF_MUL:
                return true;

        /* Shift operators range is only computable if shift dimension operand
         * is a constant. Shifts greater than 31 or 63 are undefined. This
         * includes shifts by a negative number.
         */
        case BPF_LSH:
        case BPF_RSH:
        case BPF_ARSH:
                return (src_is_const && src_reg->umax_value < insn_bitness);
        default:
                return false;
        }
}

/* WARNING: This function does calculations on 64-bit values, but the actual
 * execution may occur on 32-bit values. Therefore, things like bitshifts
 * need extra checks in the 32-bit case.
 */
static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
                                      struct bpf_insn *insn,
                                      struct bpf_reg_state *dst_reg,
                                      struct bpf_reg_state src_reg)
{
        u8 opcode = BPF_OP(insn->code);
        bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
        int ret;

        if (!is_safe_to_compute_dst_reg_range(insn, &src_reg)) {
                __mark_reg_unknown(env, dst_reg);
                return 0;
        }

        if (sanitize_needed(opcode)) {
                ret = sanitize_val_alu(env, insn);
                if (ret < 0)
                        return sanitize_err(env, insn, ret, NULL, NULL);
        }

        /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops.
         * There are two classes of instructions: The first class we track both
         * alu32 and alu64 sign/unsigned bounds independently this provides the
         * greatest amount of precision when alu operations are mixed with jmp32
         * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD,
         * and BPF_OR. This is possible because these ops have fairly easy to
         * understand and calculate behavior in both 32-bit and 64-bit alu ops.
         * See alu32 verifier tests for examples. The second class of
         * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy
         * with regards to tracking sign/unsigned bounds because the bits may
         * cross subreg boundaries in the alu64 case. When this happens we mark
         * the reg unbounded in the subreg bound space and use the resulting
         * tnum to calculate an approximation of the sign/unsigned bounds.
         */
        switch (opcode) {
        case BPF_ADD:
                scalar32_min_max_add(dst_reg, &src_reg);
                scalar_min_max_add(dst_reg, &src_reg);
                dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
                break;
        case BPF_SUB:
                scalar32_min_max_sub(dst_reg, &src_reg);
                scalar_min_max_sub(dst_reg, &src_reg);
                dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
                break;
        case BPF_MUL:
                dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
                scalar32_min_max_mul(dst_reg, &src_reg);
                scalar_min_max_mul(dst_reg, &src_reg);
                break;
        case BPF_AND:
                dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
                scalar32_min_max_and(dst_reg, &src_reg);
                scalar_min_max_and(dst_reg, &src_reg);
                break;
        case BPF_OR:
                dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
                scalar32_min_max_or(dst_reg, &src_reg);
                scalar_min_max_or(dst_reg, &src_reg);
                break;
        case BPF_XOR:
                dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
                scalar32_min_max_xor(dst_reg, &src_reg);
                scalar_min_max_xor(dst_reg, &src_reg);
                break;
        case BPF_LSH:
                if (alu32)
                        scalar32_min_max_lsh(dst_reg, &src_reg);
                else
                        scalar_min_max_lsh(dst_reg, &src_reg);
                break;
        case BPF_RSH:
                if (alu32)
                        scalar32_min_max_rsh(dst_reg, &src_reg);
                else
                        scalar_min_max_rsh(dst_reg, &src_reg);
                break;
        case BPF_ARSH:
                if (alu32)
                        scalar32_min_max_arsh(dst_reg, &src_reg);
                else
                        scalar_min_max_arsh(dst_reg, &src_reg);
                break;
        default:
                break;
        }

        /* ALU32 ops are zero extended into 64bit register */
        if (alu32)
                zext_32_to_64(dst_reg);
        reg_bounds_sync(dst_reg);
        return 0;
}

/* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
 * and var_off.
 */
static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
                                   struct bpf_insn *insn)
{
        struct bpf_verifier_state *vstate = env->cur_state;
        struct bpf_func_state *state = vstate->frame[vstate->curframe];
        struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
        struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
        bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
        u8 opcode = BPF_OP(insn->code);
        int err;

        dst_reg = &regs[insn->dst_reg];
        src_reg = NULL;

        if (dst_reg->type == PTR_TO_ARENA) {
                struct bpf_insn_aux_data *aux = cur_aux(env);

                if (BPF_CLASS(insn->code) == BPF_ALU64)
                        /*
                         * 32-bit operations zero upper bits automatically.
                         * 64-bit operations need to be converted to 32.
                         */
                        aux->needs_zext = true;

                /* Any arithmetic operations are allowed on arena pointers */
                return 0;
        }

        if (dst_reg->type != SCALAR_VALUE)
                ptr_reg = dst_reg;

        if (BPF_SRC(insn->code) == BPF_X) {
                src_reg = &regs[insn->src_reg];
                if (src_reg->type != SCALAR_VALUE) {
                        if (dst_reg->type != SCALAR_VALUE) {
                                /* Combining two pointers by any ALU op yields
                                 * an arbitrary scalar. Disallow all math except
                                 * pointer subtraction
                                 */
                                if (opcode == BPF_SUB && env->allow_ptr_leaks) {
                                        mark_reg_unknown(env, regs, insn->dst_reg);
                                        return 0;
                                }
                                verbose(env, "R%d pointer %s pointer prohibited\n",
                                        insn->dst_reg,
                                        bpf_alu_string[opcode >> 4]);
                                return -EACCES;
                        } else {
                                /* scalar += pointer
                                 * This is legal, but we have to reverse our
                                 * src/dest handling in computing the range
                                 */
                                err = mark_chain_precision(env, insn->dst_reg);
                                if (err)
                                        return err;
                                return adjust_ptr_min_max_vals(env, insn,
                                                               src_reg, dst_reg);
                        }
                } else if (ptr_reg) {
                        /* pointer += scalar */
                        err = mark_chain_precision(env, insn->src_reg);
                        if (err)
                                return err;
                        return adjust_ptr_min_max_vals(env, insn,
                                                       dst_reg, src_reg);
                } else if (dst_reg->precise) {
                        /* if dst_reg is precise, src_reg should be precise as well */
                        err = mark_chain_precision(env, insn->src_reg);
                        if (err)
                                return err;
                }
        } else {
                /* Pretend the src is a reg with a known value, since we only
                 * need to be able to read from this state.
                 */
                off_reg.type = SCALAR_VALUE;
                __mark_reg_known(&off_reg, insn->imm);
                src_reg = &off_reg;
                if (ptr_reg) /* pointer += K */
                        return adjust_ptr_min_max_vals(env, insn,
                                                       ptr_reg, src_reg);
        }

        /* Got here implies adding two SCALAR_VALUEs */
        if (WARN_ON_ONCE(ptr_reg)) {
                print_verifier_state(env, vstate, vstate->curframe, true);
                verbose(env, "verifier internal error: unexpected ptr_reg\n");
                return -EINVAL;
        }
        if (WARN_ON(!src_reg)) {
                print_verifier_state(env, vstate, vstate->curframe, true);
                verbose(env, "verifier internal error: no src_reg\n");
                return -EINVAL;
        }
        err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
        if (err)
                return err;
        /*
         * Compilers can generate the code
         * r1 = r2
         * r1 += 0x1
         * if r2 < 1000 goto ...
         * use r1 in memory access
         * So for 64-bit alu remember constant delta between r2 and r1 and
         * update r1 after 'if' condition.
         */
        if (env->bpf_capable &&
            BPF_OP(insn->code) == BPF_ADD && !alu32 &&
            dst_reg->id && is_reg_const(src_reg, false)) {
                u64 val = reg_const_value(src_reg, false);

                if ((dst_reg->id & BPF_ADD_CONST) ||
                    /* prevent overflow in sync_linked_regs() later */
                    val > (u32)S32_MAX) {
                        /*
                         * If the register already went through rX += val
                         * we cannot accumulate another val into rx->off.
                         */
                        dst_reg->off = 0;
                        dst_reg->id = 0;
                } else {
                        dst_reg->id |= BPF_ADD_CONST;
                        dst_reg->off = val;
                }
        } else {
                /*
                 * Make sure ID is cleared otherwise dst_reg min/max could be
                 * incorrectly propagated into other registers by sync_linked_regs()
                 */
                dst_reg->id = 0;
        }
        return 0;
}

/* check validity of 32-bit and 64-bit arithmetic operations */
static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
{
        struct bpf_reg_state *regs = cur_regs(env);
        u8 opcode = BPF_OP(insn->code);
        int err;

        if (opcode == BPF_END || opcode == BPF_NEG) {
                if (opcode == BPF_NEG) {
                        if (BPF_SRC(insn->code) != BPF_K ||
                            insn->src_reg != BPF_REG_0 ||
                            insn->off != 0 || insn->imm != 0) {
                                verbose(env, "BPF_NEG uses reserved fields\n");
                                return -EINVAL;
                        }
                } else {
                        if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
                            (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
                            (BPF_CLASS(insn->code) == BPF_ALU64 &&
                             BPF_SRC(insn->code) != BPF_TO_LE)) {
                                verbose(env, "BPF_END uses reserved fields\n");
                                return -EINVAL;
                        }
                }

                /* check src operand */
                err = check_reg_arg(env, insn->dst_reg, SRC_OP);
                if (err)
                        return err;

                if (is_pointer_value(env, insn->dst_reg)) {
                        verbose(env, "R%d pointer arithmetic prohibited\n",
                                insn->dst_reg);
                        return -EACCES;
                }

                /* check dest operand */
                err = check_reg_arg(env, insn->dst_reg, DST_OP);
                if (err)
                        return err;

        } else if (opcode == BPF_MOV) {

                if (BPF_SRC(insn->code) == BPF_X) {
                        if (BPF_CLASS(insn->code) == BPF_ALU) {
                                if ((insn->off != 0 && insn->off != 8 && insn->off != 16) ||
                                    insn->imm) {
                                        verbose(env, "BPF_MOV uses reserved fields\n");
                                        return -EINVAL;
                                }
                        } else if (insn->off == BPF_ADDR_SPACE_CAST) {
                                if (insn->imm != 1 && insn->imm != 1u << 16) {
                                        verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n");
                                        return -EINVAL;
                                }
                                if (!env->prog->aux->arena) {
                                        verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n");
                                        return -EINVAL;
                                }
                        } else {
                                if ((insn->off != 0 && insn->off != 8 && insn->off != 16 &&
                                     insn->off != 32) || insn->imm) {
                                        verbose(env, "BPF_MOV uses reserved fields\n");
                                        return -EINVAL;
                                }
                        }

                        /* check src operand */
                        err = check_reg_arg(env, insn->src_reg, SRC_OP);
                        if (err)
                                return err;
                } else {
                        if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
                                verbose(env, "BPF_MOV uses reserved fields\n");
                                return -EINVAL;
                        }
                }

                /* check dest operand, mark as required later */
                err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
                if (err)
                        return err;

                if (BPF_SRC(insn->code) == BPF_X) {
                        struct bpf_reg_state *src_reg = regs + insn->src_reg;
                        struct bpf_reg_state *dst_reg = regs + insn->dst_reg;

                        if (BPF_CLASS(insn->code) == BPF_ALU64) {
                                if (insn->imm) {
                                        /* off == BPF_ADDR_SPACE_CAST */
                                        mark_reg_unknown(env, regs, insn->dst_reg);
                                        if (insn->imm == 1) { /* cast from as(1) to as(0) */
                                                dst_reg->type = PTR_TO_ARENA;
                                                /* PTR_TO_ARENA is 32-bit */
                                                dst_reg->subreg_def = env->insn_idx + 1;
                                        }
                                } else if (insn->off == 0) {
                                        /* case: R1 = R2
                                         * copy register state to dest reg
                                         */
                                        assign_scalar_id_before_mov(env, src_reg);
                                        copy_register_state(dst_reg, src_reg);
                                        dst_reg->live |= REG_LIVE_WRITTEN;
                                        dst_reg->subreg_def = DEF_NOT_SUBREG;
                                } else {
                                        /* case: R1 = (s8, s16 s32)R2 */
                                        if (is_pointer_value(env, insn->src_reg)) {
                                                verbose(env,
                                                        "R%d sign-extension part of pointer\n",
                                                        insn->src_reg);
                                                return -EACCES;
                                        } else if (src_reg->type == SCALAR_VALUE) {
                                                bool no_sext;

                                                no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));
                                                if (no_sext)
                                                        assign_scalar_id_before_mov(env, src_reg);
                                                copy_register_state(dst_reg, src_reg);
                                                if (!no_sext)
                                                        dst_reg->id = 0;
                                                coerce_reg_to_size_sx(dst_reg, insn->off >> 3);
                                                dst_reg->live |= REG_LIVE_WRITTEN;
                                                dst_reg->subreg_def = DEF_NOT_SUBREG;
                                        } else {
                                                mark_reg_unknown(env, regs, insn->dst_reg);
                                        }
                                }
                        } else {
                                /* R1 = (u32) R2 */
                                if (is_pointer_value(env, insn->src_reg)) {
                                        verbose(env,
                                                "R%d partial copy of pointer\n",
                                                insn->src_reg);
                                        return -EACCES;
                                } else if (src_reg->type == SCALAR_VALUE) {
                                        if (insn->off == 0) {
                                                bool is_src_reg_u32 = get_reg_width(src_reg) <= 32;

                                                if (is_src_reg_u32)
                                                        assign_scalar_id_before_mov(env, src_reg);
                                                copy_register_state(dst_reg, src_reg);
                                                /* Make sure ID is cleared if src_reg is not in u32
                                                 * range otherwise dst_reg min/max could be incorrectly
                                                 * propagated into src_reg by sync_linked_regs()
                                                 */
                                                if (!is_src_reg_u32)
                                                        dst_reg->id = 0;
                                                dst_reg->live |= REG_LIVE_WRITTEN;
                                                dst_reg->subreg_def = env->insn_idx + 1;
                                        } else {
                                                /* case: W1 = (s8, s16)W2 */
                                                bool no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));

                                                if (no_sext)
                                                        assign_scalar_id_before_mov(env, src_reg);
                                                copy_register_state(dst_reg, src_reg);
                                                if (!no_sext)
                                                        dst_reg->id = 0;
                                                dst_reg->live |= REG_LIVE_WRITTEN;
                                                dst_reg->subreg_def = env->insn_idx + 1;
                                                coerce_subreg_to_size_sx(dst_reg, insn->off >> 3);
                                        }
                                } else {
                                        mark_reg_unknown(env, regs,
                                                         insn->dst_reg);
                                }
                                zext_32_to_64(dst_reg);
                                reg_bounds_sync(dst_reg);
                        }
                } else {
                        /* case: R = imm
                         * remember the value we stored into this reg
                         */
                        /* clear any state __mark_reg_known doesn't set */
                        mark_reg_unknown(env, regs, insn->dst_reg);
                        regs[insn->dst_reg].type = SCALAR_VALUE;
                        if (BPF_CLASS(insn->code) == BPF_ALU64) {
                                __mark_reg_known(regs + insn->dst_reg,
                                                 insn->imm);
                        } else {
                                __mark_reg_known(regs + insn->dst_reg,
                                                 (u32)insn->imm);
                        }
                }

        } else if (opcode > BPF_END) {
                verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
                return -EINVAL;

        } else {        /* all other ALU ops: and, sub, xor, add, ... */

                if (BPF_SRC(insn->code) == BPF_X) {
                        if (insn->imm != 0 || insn->off > 1 ||
                            (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
                                verbose(env, "BPF_ALU uses reserved fields\n");
                                return -EINVAL;
                        }
                        /* check src1 operand */
                        err = check_reg_arg(env, insn->src_reg, SRC_OP);
                        if (err)
                                return err;
                } else {
                        if (insn->src_reg != BPF_REG_0 || insn->off > 1 ||
                            (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
                                verbose(env, "BPF_ALU uses reserved fields\n");
                                return -EINVAL;
                        }
                }

                /* check src2 operand */
                err = check_reg_arg(env, insn->dst_reg, SRC_OP);
                if (err)
                        return err;

                if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
                    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
                        verbose(env, "div by zero\n");
                        return -EINVAL;
                }

                if ((opcode == BPF_LSH || opcode == BPF_RSH ||
                     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
                        int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;

                        if (insn->imm < 0 || insn->imm >= size) {
                                verbose(env, "invalid shift %d\n", insn->imm);
                                return -EINVAL;
                        }
                }

                /* check dest operand */
                err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
                err = err ?: adjust_reg_min_max_vals(env, insn);
                if (err)
                        return err;
        }

        return reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");
}

static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
                                   struct bpf_reg_state *dst_reg,
                                   enum bpf_reg_type type,
                                   bool range_right_open)
{
        struct bpf_func_state *state;
        struct bpf_reg_state *reg;
        int new_range;

        if (dst_reg->off < 0 ||
            (dst_reg->off == 0 && range_right_open))
                /* This doesn't give us any range */
                return;

        if (dst_reg->umax_value > MAX_PACKET_OFF ||
            dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
                /* Risk of overflow.  For instance, ptr + (1<<63) may be less
                 * than pkt_end, but that's because it's also less than pkt.
                 */
                return;

        new_range = dst_reg->off;
        if (range_right_open)
                new_range++;

        /* Examples for register markings:
         *
         * pkt_data in dst register:
         *
         *   r2 = r3;
         *   r2 += 8;
         *   if (r2 > pkt_end) goto <handle exception>
         *   <access okay>
         *
         *   r2 = r3;
         *   r2 += 8;
         *   if (r2 < pkt_end) goto <access okay>
         *   <handle exception>
         *
         *   Where:
         *     r2 == dst_reg, pkt_end == src_reg
         *     r2=pkt(id=n,off=8,r=0)
         *     r3=pkt(id=n,off=0,r=0)
         *
         * pkt_data in src register:
         *
         *   r2 = r3;
         *   r2 += 8;
         *   if (pkt_end >= r2) goto <access okay>
         *   <handle exception>
         *
         *   r2 = r3;
         *   r2 += 8;
         *   if (pkt_end <= r2) goto <handle exception>
         *   <access okay>
         *
         *   Where:
         *     pkt_end == dst_reg, r2 == src_reg
         *     r2=pkt(id=n,off=8,r=0)
         *     r3=pkt(id=n,off=0,r=0)
         *
         * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
         * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
         * and [r3, r3 + 8-1) respectively is safe to access depending on
         * the check.
         */

        /* If our ids match, then we must have the same max_value.  And we
         * don't care about the other reg's fixed offset, since if it's too big
         * the range won't allow anything.
         * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
         */
        bpf_for_each_reg_in_vstate(vstate, state, reg, ({
                if (reg->type == type && reg->id == dst_reg->id)
                        /* keep the maximum range already checked */
                        reg->range = max(reg->range, new_range);
        }));
}

/*
 * <reg1> <op> <reg2>, currently assuming reg2 is a constant
 */
static int is_scalar_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
                                  u8 opcode, bool is_jmp32)
{
        struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off;
        struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off;
        u64 umin1 = is_jmp32 ? (u64)reg1->u32_min_value : reg1->umin_value;
        u64 umax1 = is_jmp32 ? (u64)reg1->u32_max_value : reg1->umax_value;
        s64 smin1 = is_jmp32 ? (s64)reg1->s32_min_value : reg1->smin_value;
        s64 smax1 = is_jmp32 ? (s64)reg1->s32_max_value : reg1->smax_value;
        u64 umin2 = is_jmp32 ? (u64)reg2->u32_min_value : reg2->umin_value;
        u64 umax2 = is_jmp32 ? (u64)reg2->u32_max_value : reg2->umax_value;
        s64 smin2 = is_jmp32 ? (s64)reg2->s32_min_value : reg2->smin_value;
        s64 smax2 = is_jmp32 ? (s64)reg2->s32_max_value : reg2->smax_value;

        switch (opcode) {
        case BPF_JEQ:
                /* constants, umin/umax and smin/smax checks would be
                 * redundant in this case because they all should match
                 */
                if (tnum_is_const(t1) && tnum_is_const(t2))
                        return t1.value == t2.value;
                /* non-overlapping ranges */
                if (umin1 > umax2 || umax1 < umin2)
                        return 0;
                if (smin1 > smax2 || smax1 < smin2)
                        return 0;
                if (!is_jmp32) {
                        /* if 64-bit ranges are inconclusive, see if we can
                         * utilize 32-bit subrange knowledge to eliminate
                         * branches that can't be taken a priori
                         */
                        if (reg1->u32_min_value > reg2->u32_max_value ||
                            reg1->u32_max_value < reg2->u32_min_value)
                                return 0;
                        if (reg1->s32_min_value > reg2->s32_max_value ||
                            reg1->s32_max_value < reg2->s32_min_value)
                                return 0;
                }
                break;
        case BPF_JNE:
                /* constants, umin/umax and smin/smax checks would be
                 * redundant in this case because they all should match
                 */
                if (tnum_is_const(t1) && tnum_is_const(t2))
                        return t1.value != t2.value;
                /* non-overlapping ranges */
                if (umin1 > umax2 || umax1 < umin2)
                        return 1;
                if (smin1 > smax2 || smax1 < smin2)
                        return 1;
                if (!is_jmp32) {
                        /* if 64-bit ranges are inconclusive, see if we can
                         * utilize 32-bit subrange knowledge to eliminate
                         * branches that can't be taken a priori
                         */
                        if (reg1->u32_min_value > reg2->u32_max_value ||
                            reg1->u32_max_value < reg2->u32_min_value)
                                return 1;
                        if (reg1->s32_min_value > reg2->s32_max_value ||
                            reg1->s32_max_value < reg2->s32_min_value)
                                return 1;
                }
                break;
        case BPF_JSET:
                if (!is_reg_const(reg2, is_jmp32)) {
                        swap(reg1, reg2);
                        swap(t1, t2);
                }
                if (!is_reg_const(reg2, is_jmp32))
                        return -1;
                if ((~t1.mask & t1.value) & t2.value)
                        return 1;
                if (!((t1.mask | t1.value) & t2.value))
                        return 0;
                break;
        case BPF_JGT:
                if (umin1 > umax2)
                        return 1;
                else if (umax1 <= umin2)
                        return 0;
                break;
        case BPF_JSGT:
                if (smin1 > smax2)
                        return 1;
                else if (smax1 <= smin2)
                        return 0;
                break;
        case BPF_JLT:
                if (umax1 < umin2)
                        return 1;
                else if (umin1 >= umax2)
                        return 0;
                break;
        case BPF_JSLT:
                if (smax1 < smin2)
                        return 1;
                else if (smin1 >= smax2)
                        return 0;
                break;
        case BPF_JGE:
                if (umin1 >= umax2)
                        return 1;
                else if (umax1 < umin2)
                        return 0;
                break;
        case BPF_JSGE:
                if (smin1 >= smax2)
                        return 1;
                else if (smax1 < smin2)
                        return 0;
                break;
        case BPF_JLE:
                if (umax1 <= umin2)
                        return 1;
                else if (umin1 > umax2)
                        return 0;
                break;
        case BPF_JSLE:
                if (smax1 <= smin2)
                        return 1;
                else if (smin1 > smax2)
                        return 0;
                break;
        }

        return -1;
}

static int flip_opcode(u32 opcode)
{
        /* How can we transform "a <op> b" into "b <op> a"? */
        static const u8 opcode_flip[16] = {
                /* these stay the same */
                [BPF_JEQ  >> 4] = BPF_JEQ,
                [BPF_JNE  >> 4] = BPF_JNE,
                [BPF_JSET >> 4] = BPF_JSET,
                /* these swap "lesser" and "greater" (L and G in the opcodes) */
                [BPF_JGE  >> 4] = BPF_JLE,
                [BPF_JGT  >> 4] = BPF_JLT,
                [BPF_JLE  >> 4] = BPF_JGE,
                [BPF_JLT  >> 4] = BPF_JGT,
                [BPF_JSGE >> 4] = BPF_JSLE,
                [BPF_JSGT >> 4] = BPF_JSLT,
                [BPF_JSLE >> 4] = BPF_JSGE,
                [BPF_JSLT >> 4] = BPF_JSGT
        };
        return opcode_flip[opcode >> 4];
}

static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
                                   struct bpf_reg_state *src_reg,
                                   u8 opcode)
{
        struct bpf_reg_state *pkt;

        if (src_reg->type == PTR_TO_PACKET_END) {
                pkt = dst_reg;
        } else if (dst_reg->type == PTR_TO_PACKET_END) {
                pkt = src_reg;
                opcode = flip_opcode(opcode);
        } else {
                return -1;
        }

        if (pkt->range >= 0)
                return -1;

        switch (opcode) {
        case BPF_JLE:
                /* pkt <= pkt_end */
                fallthrough;
        case BPF_JGT:
                /* pkt > pkt_end */
                if (pkt->range == BEYOND_PKT_END)
                        /* pkt has at last one extra byte beyond pkt_end */
                        return opcode == BPF_JGT;
                break;
        case BPF_JLT:
                /* pkt < pkt_end */
                fallthrough;
        case BPF_JGE:
                /* pkt >= pkt_end */
                if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END)
                        return opcode == BPF_JGE;
                break;
        }
        return -1;
}

/* compute branch direction of the expression "if (<reg1> opcode <reg2>) goto target;"
 * and return:
 *  1 - branch will be taken and "goto target" will be executed
 *  0 - branch will not be taken and fall-through to next insn
 * -1 - unknown. Example: "if (reg1 < 5)" is unknown when register value
 *      range [0,10]
 */
static int is_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
                           u8 opcode, bool is_jmp32)
{
        if (reg_is_pkt_pointer_any(reg1) && reg_is_pkt_pointer_any(reg2) && !is_jmp32)
                return is_pkt_ptr_branch_taken(reg1, reg2, opcode);

        if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) {
                u64 val;

                /* arrange that reg2 is a scalar, and reg1 is a pointer */
                if (!is_reg_const(reg2, is_jmp32)) {
                        opcode = flip_opcode(opcode);
                        swap(reg1, reg2);
                }
                /* and ensure that reg2 is a constant */
                if (!is_reg_const(reg2, is_jmp32))
                        return -1;

                if (!reg_not_null(reg1))
                        return -1;

                /* If pointer is valid tests against zero will fail so we can
                 * use this to direct branch taken.
                 */
                val = reg_const_value(reg2, is_jmp32);
                if (val != 0)
                        return -1;

                switch (opcode) {
                case BPF_JEQ:
                        return 0;
                case BPF_JNE:
                        return 1;
                default:
                        return -1;
                }
        }

        /* now deal with two scalars, but not necessarily constants */
        return is_scalar_branch_taken(reg1, reg2, opcode, is_jmp32);
}

/* Opcode that corresponds to a *false* branch condition.
 * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2
 */
static u8 rev_opcode(u8 opcode)
{
        switch (opcode) {
        case BPF_JEQ:                return BPF_JNE;
        case BPF_JNE:                return BPF_JEQ;
        /* JSET doesn't have it's reverse opcode in BPF, so add
         * BPF_X flag to denote the reverse of that operation
         */
        case BPF_JSET:                return BPF_JSET | BPF_X;
        case BPF_JSET | BPF_X:        return BPF_JSET;
        case BPF_JGE:                return BPF_JLT;
        case BPF_JGT:                return BPF_JLE;
        case BPF_JLE:                return BPF_JGT;
        case BPF_JLT:                return BPF_JGE;
        case BPF_JSGE:                return BPF_JSLT;
        case BPF_JSGT:                return BPF_JSLE;
        case BPF_JSLE:                return BPF_JSGT;
        case BPF_JSLT:                return BPF_JSGE;
        default:                return 0;
        }
}

/* Refine range knowledge for <reg1> <op> <reg>2 conditional operation. */
static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
                                u8 opcode, bool is_jmp32)
{
        struct tnum t;
        u64 val;

        /* In case of GE/GT/SGE/JST, reuse LE/LT/SLE/SLT logic from below */
        switch (opcode) {
        case BPF_JGE:
        case BPF_JGT:
        case BPF_JSGE:
        case BPF_JSGT:
                opcode = flip_opcode(opcode);
                swap(reg1, reg2);
                break;
        default:
                break;
        }

        switch (opcode) {
        case BPF_JEQ:
                if (is_jmp32) {
                        reg1->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
                        reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
                        reg1->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
                        reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
                        reg2->u32_min_value = reg1->u32_min_value;
                        reg2->u32_max_value = reg1->u32_max_value;
                        reg2->s32_min_value = reg1->s32_min_value;
                        reg2->s32_max_value = reg1->s32_max_value;

                        t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off));
                        reg1->var_off = tnum_with_subreg(reg1->var_off, t);
                        reg2->var_off = tnum_with_subreg(reg2->var_off, t);
                } else {
                        reg1->umin_value = max(reg1->umin_value, reg2->umin_value);
                        reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
                        reg1->smin_value = max(reg1->smin_value, reg2->smin_value);
                        reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
                        reg2->umin_value = reg1->umin_value;
                        reg2->umax_value = reg1->umax_value;
                        reg2->smin_value = reg1->smin_value;
                        reg2->smax_value = reg1->smax_value;

                        reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off);
                        reg2->var_off = reg1->var_off;
                }
                break;
        case BPF_JNE:
                if (!is_reg_const(reg2, is_jmp32))
                        swap(reg1, reg2);
                if (!is_reg_const(reg2, is_jmp32))
                        break;

                /* try to recompute the bound of reg1 if reg2 is a const and
                 * is exactly the edge of reg1.
                 */
                val = reg_const_value(reg2, is_jmp32);
                if (is_jmp32) {
                        /* u32_min_value is not equal to 0xffffffff at this point,
                         * because otherwise u32_max_value is 0xffffffff as well,
                         * in such a case both reg1 and reg2 would be constants,
                         * jump would be predicted and reg_set_min_max() won't
                         * be called.
                         *
                         * Same reasoning works for all {u,s}{min,max}{32,64} cases
                         * below.
                         */
                        if (reg1->u32_min_value == (u32)val)
                                reg1->u32_min_value++;
                        if (reg1->u32_max_value == (u32)val)
                                reg1->u32_max_value--;
                        if (reg1->s32_min_value == (s32)val)
                                reg1->s32_min_value++;
                        if (reg1->s32_max_value == (s32)val)
                                reg1->s32_max_value--;
                } else {
                        if (reg1->umin_value == (u64)val)
                                reg1->umin_value++;
                        if (reg1->umax_value == (u64)val)
                                reg1->umax_value--;
                        if (reg1->smin_value == (s64)val)
                                reg1->smin_value++;
                        if (reg1->smax_value == (s64)val)
                                reg1->smax_value--;
                }
                break;
        case BPF_JSET:
                if (!is_reg_const(reg2, is_jmp32))
                        swap(reg1, reg2);
                if (!is_reg_const(reg2, is_jmp32))
                        break;
                val = reg_const_value(reg2, is_jmp32);
                /* BPF_JSET (i.e., TRUE branch, *not* BPF_JSET | BPF_X)
                 * requires single bit to learn something useful. E.g., if we
                 * know that `r1 & 0x3` is true, then which bits (0, 1, or both)
                 * are actually set? We can learn something definite only if
                 * it's a single-bit value to begin with.
                 *
                 * BPF_JSET | BPF_X (i.e., negation of BPF_JSET) doesn't have
                 * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor
                 * bit 1 is set, which we can readily use in adjustments.
                 */
                if (!is_power_of_2(val))
                        break;
                if (is_jmp32) {
                        t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val));
                        reg1->var_off = tnum_with_subreg(reg1->var_off, t);
                } else {
                        reg1->var_off = tnum_or(reg1->var_off, tnum_const(val));
                }
                break;
        case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */
                if (!is_reg_const(reg2, is_jmp32))
                        swap(reg1, reg2);
                if (!is_reg_const(reg2, is_jmp32))
                        break;
                val = reg_const_value(reg2, is_jmp32);
                if (is_jmp32) {
                        t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val));
                        reg1->var_off = tnum_with_subreg(reg1->var_off, t);
                } else {
                        reg1->var_off = tnum_and(reg1->var_off, tnum_const(~val));
                }
                break;
        case BPF_JLE:
                if (is_jmp32) {
                        reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
                        reg2->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
                } else {
                        reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
                        reg2->umin_value = max(reg1->umin_value, reg2->umin_value);
                }
                break;
        case BPF_JLT:
                if (is_jmp32) {
                        reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value - 1);
                        reg2->u32_min_value = max(reg1->u32_min_value + 1, reg2->u32_min_value);
                } else {
                        reg1->umax_value = min(reg1->umax_value, reg2->umax_value - 1);
                        reg2->umin_value = max(reg1->umin_value + 1, reg2->umin_value);
                }
                break;
        case BPF_JSLE:
                if (is_jmp32) {
                        reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
                        reg2->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
                } else {
                        reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
                        reg2->smin_value = max(reg1->smin_value, reg2->smin_value);
                }
                break;
        case BPF_JSLT:
                if (is_jmp32) {
                        reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value - 1);
                        reg2->s32_min_value = max(reg1->s32_min_value + 1, reg2->s32_min_value);
                } else {
                        reg1->smax_value = min(reg1->smax_value, reg2->smax_value - 1);
                        reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value);
                }
                break;
        default:
                return;
        }
}

/* Adjusts the register min/max values in the case that the dst_reg and
 * src_reg are both SCALAR_VALUE registers (or we are simply doing a BPF_K
 * check, in which case we have a fake SCALAR_VALUE representing insn->imm).
 * Technically we can do similar adjustments for pointers to the same object,
 * but we don't support that right now.
 */
static int reg_set_min_max(struct bpf_verifier_env *env,
                           struct bpf_reg_state *true_reg1,
                           struct bpf_reg_state *true_reg2,
                           struct bpf_reg_state *false_reg1,
                           struct bpf_reg_state *false_reg2,
                           u8 opcode, bool is_jmp32)
{
        int err;

        /* If either register is a pointer, we can't learn anything about its
         * variable offset from the compare (unless they were a pointer into
         * the same object, but we don't bother with that).
         */
        if (false_reg1->type != SCALAR_VALUE || false_reg2->type != SCALAR_VALUE)
                return 0;

        /* fallthrough (FALSE) branch */
        regs_refine_cond_op(false_reg1, false_reg2, rev_opcode(opcode), is_jmp32);
        reg_bounds_sync(false_reg1);
        reg_bounds_sync(false_reg2);

        /* jump (TRUE) branch */
        regs_refine_cond_op(true_reg1, true_reg2, opcode, is_jmp32);
        reg_bounds_sync(true_reg1);
        reg_bounds_sync(true_reg2);

        err = reg_bounds_sanity_check(env, true_reg1, "true_reg1");
        err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2");
        err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1");
        err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2");
        return err;
}

static void mark_ptr_or_null_reg(struct bpf_func_state *state,
                                 struct bpf_reg_state *reg, u32 id,
                                 bool is_null)
{
        if (type_may_be_null(reg->type) && reg->id == id &&
            (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) {
                /* Old offset (both fixed and variable parts) should have been
                 * known-zero, because we don't allow pointer arithmetic on
                 * pointers that might be NULL. If we see this happening, don't
                 * convert the register.
                 *
                 * But in some cases, some helpers that return local kptrs
                 * advance offset for the returned pointer. In those cases, it
                 * is fine to expect to see reg->off.
                 */
                if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0)))
                        return;
                if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) &&
                    WARN_ON_ONCE(reg->off))
                        return;

                if (is_null) {
                        reg->type = SCALAR_VALUE;
                        /* We don't need id and ref_obj_id from this point
                         * onwards anymore, thus we should better reset it,
                         * so that state pruning has chances to take effect.
                         */
                        reg->id = 0;
                        reg->ref_obj_id = 0;

                        return;
                }

                mark_ptr_not_null_reg(reg);

                if (!reg_may_point_to_spin_lock(reg)) {
                        /* For not-NULL ptr, reg->ref_obj_id will be reset
                         * in release_reference().
                         *
                         * reg->id is still used by spin_lock ptr. Other
                         * than spin_lock ptr type, reg->id can be reset.
                         */
                        reg->id = 0;
                }
        }
}

/* The logic is similar to find_good_pkt_pointers(), both could eventually
 * be folded together at some point.
 */
static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
                                  bool is_null)
{
        struct bpf_func_state *state = vstate->frame[vstate->curframe];
        struct bpf_reg_state *regs = state->regs, *reg;
        u32 ref_obj_id = regs[regno].ref_obj_id;
        u32 id = regs[regno].id;

        if (ref_obj_id && ref_obj_id == id && is_null)
                /* regs[regno] is in the " == NULL" branch.
                 * No one could have freed the reference state before
                 * doing the NULL check.
                 */
                WARN_ON_ONCE(release_reference_nomark(vstate, id));

        bpf_for_each_reg_in_vstate(vstate, state, reg, ({
                mark_ptr_or_null_reg(state, reg, id, is_null);
        }));
}

static bool try_match_pkt_pointers(const struct bpf_insn *insn,
                                   struct bpf_reg_state *dst_reg,
                                   struct bpf_reg_state *src_reg,
                                   struct bpf_verifier_state *this_branch,
                                   struct bpf_verifier_state *other_branch)
{
        if (BPF_SRC(insn->code) != BPF_X)
                return false;

        /* Pointers are always 64-bit. */
        if (BPF_CLASS(insn->code) == BPF_JMP32)
                return false;

        switch (BPF_OP(insn->code)) {
        case BPF_JGT:
                if ((dst_reg->type == PTR_TO_PACKET &&
                     src_reg->type == PTR_TO_PACKET_END) ||
                    (dst_reg->type == PTR_TO_PACKET_META &&
                     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
                        /* pkt_data' > pkt_end, pkt_meta' > pkt_data */
                        find_good_pkt_pointers(this_branch, dst_reg,
                                               dst_reg->type, false);
                        mark_pkt_end(other_branch, insn->dst_reg, true);
                } else if ((dst_reg->type == PTR_TO_PACKET_END &&
                            src_reg->type == PTR_TO_PACKET) ||
                           (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
                            src_reg->type == PTR_TO_PACKET_META)) {
                        /* pkt_end > pkt_data', pkt_data > pkt_meta' */
                        find_good_pkt_pointers(other_branch, src_reg,
                                               src_reg->type, true);
                        mark_pkt_end(this_branch, insn->src_reg, false);
                } else {
                        return false;
                }
                break;
        case BPF_JLT:
                if ((dst_reg->type == PTR_TO_PACKET &&
                     src_reg->type == PTR_TO_PACKET_END) ||
                    (dst_reg->type == PTR_TO_PACKET_META &&
                     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
                        /* pkt_data' < pkt_end, pkt_meta' < pkt_data */
                        find_good_pkt_pointers(other_branch, dst_reg,
                                               dst_reg->type, true);
                        mark_pkt_end(this_branch, insn->dst_reg, false);
                } else if ((dst_reg->type == PTR_TO_PACKET_END &&
                            src_reg->type == PTR_TO_PACKET) ||
                           (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
                            src_reg->type == PTR_TO_PACKET_META)) {
                        /* pkt_end < pkt_data', pkt_data > pkt_meta' */
                        find_good_pkt_pointers(this_branch, src_reg,
                                               src_reg->type, false);
                        mark_pkt_end(other_branch, insn->src_reg, true);
                } else {
                        return false;
                }
                break;
        case BPF_JGE:
                if ((dst_reg->type == PTR_TO_PACKET &&
                     src_reg->type == PTR_TO_PACKET_END) ||
                    (dst_reg->type == PTR_TO_PACKET_META &&
                     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
                        /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
                        find_good_pkt_pointers(this_branch, dst_reg,
                                               dst_reg->type, true);
                        mark_pkt_end(other_branch, insn->dst_reg, false);
                } else if ((dst_reg->type == PTR_TO_PACKET_END &&
                            src_reg->type == PTR_TO_PACKET) ||
                           (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
                            src_reg->type == PTR_TO_PACKET_META)) {
                        /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
                        find_good_pkt_pointers(other_branch, src_reg,
                                               src_reg->type, false);
                        mark_pkt_end(this_branch, insn->src_reg, true);
                } else {
                        return false;
                }
                break;
        case BPF_JLE:
                if ((dst_reg->type == PTR_TO_PACKET &&
                     src_reg->type == PTR_TO_PACKET_END) ||
                    (dst_reg->type == PTR_TO_PACKET_META &&
                     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
                        /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
                        find_good_pkt_pointers(other_branch, dst_reg,
                                               dst_reg->type, false);
                        mark_pkt_end(this_branch, insn->dst_reg, true);
                } else if ((dst_reg->type == PTR_TO_PACKET_END &&
                            src_reg->type == PTR_TO_PACKET) ||
                           (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
                            src_reg->type == PTR_TO_PACKET_META)) {
                        /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
                        find_good_pkt_pointers(this_branch, src_reg,
                                               src_reg->type, true);
                        mark_pkt_end(other_branch, insn->src_reg, false);
                } else {
                        return false;
                }
                break;
        default:
                return false;
        }

        return true;
}

static void __collect_linked_regs(struct linked_regs *reg_set, struct bpf_reg_state *reg,
                                  u32 id, u32 frameno, u32 spi_or_reg, bool is_reg)
{
        struct linked_reg *e;

        if (reg->type != SCALAR_VALUE || (reg->id & ~BPF_ADD_CONST) != id)
                return;

        e = linked_regs_push(reg_set);
        if (e) {
                e->frameno = frameno;
                e->is_reg = is_reg;
                e->regno = spi_or_reg;
        } else {
                reg->id = 0;
        }
}

/* For all R being scalar registers or spilled scalar registers
 * in verifier state, save R in linked_regs if R->id == id.
 * If there are too many Rs sharing same id, reset id for leftover Rs.
 */
static void collect_linked_regs(struct bpf_verifier_state *vstate, u32 id,
                                struct linked_regs *linked_regs)
{
        struct bpf_func_state *func;
        struct bpf_reg_state *reg;
        int i, j;

        id = id & ~BPF_ADD_CONST;
        for (i = vstate->curframe; i >= 0; i--) {
                func = vstate->frame[i];
                for (j = 0; j < BPF_REG_FP; j++) {
                        reg = &func->regs[j];
                        __collect_linked_regs(linked_regs, reg, id, i, j, true);
                }
                for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
                        if (!is_spilled_reg(&func->stack[j]))
                                continue;
                        reg = &func->stack[j].spilled_ptr;
                        __collect_linked_regs(linked_regs, reg, id, i, j, false);
                }
        }
}

/* For all R in linked_regs, copy known_reg range into R
 * if R->id == known_reg->id.
 */
static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_state *known_reg,
                             struct linked_regs *linked_regs)
{
        struct bpf_reg_state fake_reg;
        struct bpf_reg_state *reg;
        struct linked_reg *e;
        int i;

        for (i = 0; i < linked_regs->cnt; ++i) {
                e = &linked_regs->entries[i];
                reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno]
                                : &vstate->frame[e->frameno]->stack[e->spi].spilled_ptr;
                if (reg->type != SCALAR_VALUE || reg == known_reg)
                        continue;
                if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST))
                        continue;
                if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) ||
                    reg->off == known_reg->off) {
                        s32 saved_subreg_def = reg->subreg_def;

                        copy_register_state(reg, known_reg);
                        reg->subreg_def = saved_subreg_def;
                } else {
                        s32 saved_subreg_def = reg->subreg_def;
                        s32 saved_off = reg->off;

                        fake_reg.type = SCALAR_VALUE;
                        __mark_reg_known(&fake_reg, (s32)reg->off - (s32)known_reg->off);

                        /* reg = known_reg; reg += delta */
                        copy_register_state(reg, known_reg);
                        /*
                         * Must preserve off, id and add_const flag,
                         * otherwise another sync_linked_regs() will be incorrect.
                         */
                        reg->off = saved_off;
                        reg->subreg_def = saved_subreg_def;

                        scalar32_min_max_add(reg, &fake_reg);
                        scalar_min_max_add(reg, &fake_reg);
                        reg->var_off = tnum_add(reg->var_off, fake_reg.var_off);
                }
        }
}

static int check_cond_jmp_op(struct bpf_verifier_env *env,
                             struct bpf_insn *insn, int *insn_idx)
{
        struct bpf_verifier_state *this_branch = env->cur_state;
        struct bpf_verifier_state *other_branch;
        struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
        struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
        struct bpf_reg_state *eq_branch_regs;
        struct linked_regs linked_regs = {};
        u8 opcode = BPF_OP(insn->code);
        bool is_jmp32;
        int pred = -1;
        int err;

        /* Only conditional jumps are expected to reach here. */
        if (opcode == BPF_JA || opcode > BPF_JCOND) {
                verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
                return -EINVAL;
        }

        if (opcode == BPF_JCOND) {
                struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
                int idx = *insn_idx;

                if (insn->code != (BPF_JMP | BPF_JCOND) ||
                    insn->src_reg != BPF_MAY_GOTO ||
                    insn->dst_reg || insn->imm) {
                        verbose(env, "invalid may_goto imm %d\n", insn->imm);
                        return -EINVAL;
                }
                prev_st = find_prev_entry(env, cur_st->parent, idx);

                /* branch out 'fallthrough' insn as a new state to explore */
                queued_st = push_stack(env, idx + 1, idx, false);
                if (!queued_st)
                        return -ENOMEM;

                queued_st->may_goto_depth++;
                if (prev_st)
                        widen_imprecise_scalars(env, prev_st, queued_st);
                *insn_idx += insn->off;
                return 0;
        }

        /* check src2 operand */
        err = check_reg_arg(env, insn->dst_reg, SRC_OP);
        if (err)
                return err;

        dst_reg = &regs[insn->dst_reg];
        if (BPF_SRC(insn->code) == BPF_X) {
                if (insn->imm != 0) {
                        verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
                        return -EINVAL;
                }

                /* check src1 operand */
                err = check_reg_arg(env, insn->src_reg, SRC_OP);
                if (err)
                        return err;

                src_reg = &regs[insn->src_reg];
                if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) &&
                    is_pointer_value(env, insn->src_reg)) {
                        verbose(env, "R%d pointer comparison prohibited\n",
                                insn->src_reg);
                        return -EACCES;
                }
        } else {
                if (insn->src_reg != BPF_REG_0) {
                        verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
                        return -EINVAL;
                }
                src_reg = &env->fake_reg[0];
                memset(src_reg, 0, sizeof(*src_reg));
                src_reg->type = SCALAR_VALUE;
                __mark_reg_known(src_reg, insn->imm);
        }

        is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
        pred = is_branch_taken(dst_reg, src_reg, opcode, is_jmp32);
        if (pred >= 0) {
                /* If we get here with a dst_reg pointer type it is because
                 * above is_branch_taken() special cased the 0 comparison.
                 */
                if (!__is_pointer_value(false, dst_reg))
                        err = mark_chain_precision(env, insn->dst_reg);
                if (BPF_SRC(insn->code) == BPF_X && !err &&
                    !__is_pointer_value(false, src_reg))
                        err = mark_chain_precision(env, insn->src_reg);
                if (err)
                        return err;
        }

        if (pred == 1) {
                /* Only follow the goto, ignore fall-through. If needed, push
                 * the fall-through branch for simulation under speculative
                 * execution.
                 */
                if (!env->bypass_spec_v1 &&
                    !sanitize_speculative_path(env, insn, *insn_idx + 1,
                                               *insn_idx))
                        return -EFAULT;
                if (env->log.level & BPF_LOG_LEVEL)
                        print_insn_state(env, this_branch, this_branch->curframe);
                *insn_idx += insn->off;
                return 0;
        } else if (pred == 0) {
                /* Only follow the fall-through branch, since that's where the
                 * program will go. If needed, push the goto branch for
                 * simulation under speculative execution.
                 */
                if (!env->bypass_spec_v1 &&
                    !sanitize_speculative_path(env, insn,
                                               *insn_idx + insn->off + 1,
                                               *insn_idx))
                        return -EFAULT;
                if (env->log.level & BPF_LOG_LEVEL)
                        print_insn_state(env, this_branch, this_branch->curframe);
                return 0;
        }

        /* Push scalar registers sharing same ID to jump history,
         * do this before creating 'other_branch', so that both
         * 'this_branch' and 'other_branch' share this history
         * if parent state is created.
         */
        if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id)
                collect_linked_regs(this_branch, src_reg->id, &linked_regs);
        if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
                collect_linked_regs(this_branch, dst_reg->id, &linked_regs);
        if (linked_regs.cnt > 1) {
                err = push_insn_history(env, this_branch, 0, linked_regs_pack(&linked_regs));
                if (err)
                        return err;
        }

        other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
                                  false);
        if (!other_branch)
                return -EFAULT;
        other_branch_regs = other_branch->frame[other_branch->curframe]->regs;

        if (BPF_SRC(insn->code) == BPF_X) {
                err = reg_set_min_max(env,
                                      &other_branch_regs[insn->dst_reg],
                                      &other_branch_regs[insn->src_reg],
                                      dst_reg, src_reg, opcode, is_jmp32);
        } else /* BPF_SRC(insn->code) == BPF_K */ {
                /* reg_set_min_max() can mangle the fake_reg. Make a copy
                 * so that these are two different memory locations. The
                 * src_reg is not used beyond here in context of K.
                 */
                memcpy(&env->fake_reg[1], &env->fake_reg[0],
                       sizeof(env->fake_reg[0]));
                err = reg_set_min_max(env,
                                      &other_branch_regs[insn->dst_reg],
                                      &env->fake_reg[0],
                                      dst_reg, &env->fake_reg[1],
                                      opcode, is_jmp32);
        }
        if (err)
                return err;

        if (BPF_SRC(insn->code) == BPF_X &&
            src_reg->type == SCALAR_VALUE && src_reg->id &&
            !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
                sync_linked_regs(this_branch, src_reg, &linked_regs);
                sync_linked_regs(other_branch, &other_branch_regs[insn->src_reg], &linked_regs);
        }
        if (dst_reg->type == SCALAR_VALUE && dst_reg->id &&
            !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) {
                sync_linked_regs(this_branch, dst_reg, &linked_regs);
                sync_linked_regs(other_branch, &other_branch_regs[insn->dst_reg], &linked_regs);
        }

        /* if one pointer register is compared to another pointer
         * register check if PTR_MAYBE_NULL could be lifted.
         * E.g. register A - maybe null
         *      register B - not null
         * for JNE A, B, ... - A is not null in the false branch;
         * for JEQ A, B, ... - A is not null in the true branch.
         *
         * Since PTR_TO_BTF_ID points to a kernel struct that does
         * not need to be null checked by the BPF program, i.e.,
         * could be null even without PTR_MAYBE_NULL marking, so
         * only propagate nullness when neither reg is that type.
         */
        if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X &&
            __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) &&
            type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) &&
            base_type(src_reg->type) != PTR_TO_BTF_ID &&
            base_type(dst_reg->type) != PTR_TO_BTF_ID) {
                eq_branch_regs = NULL;
                switch (opcode) {
                case BPF_JEQ:
                        eq_branch_regs = other_branch_regs;
                        break;
                case BPF_JNE:
                        eq_branch_regs = regs;
                        break;
                default:
                        /* do nothing */
                        break;
                }
                if (eq_branch_regs) {
                        if (type_may_be_null(src_reg->type))
                                mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]);
                        else
                                mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]);
                }
        }

        /* detect if R == 0 where R is returned from bpf_map_lookup_elem().
         * NOTE: these optimizations below are related with pointer comparison
         *       which will never be JMP32.
         */
        if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
            insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
            type_may_be_null(dst_reg->type)) {
                /* Mark all identical registers in each branch as either
                 * safe or unknown depending R == 0 or R != 0 conditional.
                 */
                mark_ptr_or_null_regs(this_branch, insn->dst_reg,
                                      opcode == BPF_JNE);
                mark_ptr_or_null_regs(other_branch, insn->dst_reg,
                                      opcode == BPF_JEQ);
        } else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
                                           this_branch, other_branch) &&
                   is_pointer_value(env, insn->dst_reg)) {
                verbose(env, "R%d pointer comparison prohibited\n",
                        insn->dst_reg);
                return -EACCES;
        }
        if (env->log.level & BPF_LOG_LEVEL)
                print_insn_state(env, this_branch, this_branch->curframe);
        return 0;
}

/* verify BPF_LD_IMM64 instruction */
static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
{
        struct bpf_insn_aux_data *aux = cur_aux(env);
        struct bpf_reg_state *regs = cur_regs(env);
        struct bpf_reg_state *dst_reg;
        struct bpf_map *map;
        int err;

        if (BPF_SIZE(insn->code) != BPF_DW) {
                verbose(env, "invalid BPF_LD_IMM insn\n");
                return -EINVAL;
        }
        if (insn->off != 0) {
                verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
                return -EINVAL;
        }

        err = check_reg_arg(env, insn->dst_reg, DST_OP);
        if (err)
                return err;

        dst_reg = &regs[insn->dst_reg];
        if (insn->src_reg == 0) {
                u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;

                dst_reg->type = SCALAR_VALUE;
                __mark_reg_known(&regs[insn->dst_reg], imm);
                return 0;
        }

        /* All special src_reg cases are listed below. From this point onwards
         * we either succeed and assign a corresponding dst_reg->type after
         * zeroing the offset, or fail and reject the program.
         */
        mark_reg_known_zero(env, regs, insn->dst_reg);

        if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
                dst_reg->type = aux->btf_var.reg_type;
                switch (base_type(dst_reg->type)) {
                case PTR_TO_MEM:
                        dst_reg->mem_size = aux->btf_var.mem_size;
                        break;
                case PTR_TO_BTF_ID:
                        dst_reg->btf = aux->btf_var.btf;
                        dst_reg->btf_id = aux->btf_var.btf_id;
                        break;
                default:
                        verbose(env, "bpf verifier is misconfigured\n");
                        return -EFAULT;
                }
                return 0;
        }

        if (insn->src_reg == BPF_PSEUDO_FUNC) {
                struct bpf_prog_aux *aux = env->prog->aux;
                u32 subprogno = find_subprog(env,
                                             env->insn_idx + insn->imm + 1);

                if (!aux->func_info) {
                        verbose(env, "missing btf func_info\n");
                        return -EINVAL;
                }
                if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) {
                        verbose(env, "callback function not static\n");
                        return -EINVAL;
                }

                dst_reg->type = PTR_TO_FUNC;
                dst_reg->subprogno = subprogno;
                return 0;
        }

        map = env->used_maps[aux->map_index];
        dst_reg->map_ptr = map;

        if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
            insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
                if (map->map_type == BPF_MAP_TYPE_ARENA) {
                        __mark_reg_unknown(env, dst_reg);
                        return 0;
                }
                dst_reg->type = PTR_TO_MAP_VALUE;
                dst_reg->off = aux->map_off;
                WARN_ON_ONCE(map->max_entries != 1);
                /* We want reg->id to be same (0) as map_value is not distinct */
        } else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
                   insn->src_reg == BPF_PSEUDO_MAP_IDX) {
                dst_reg->type = CONST_PTR_TO_MAP;
        } else {
                verbose(env, "bpf verifier is misconfigured\n");
                return -EINVAL;
        }

        return 0;
}

static bool may_access_skb(enum bpf_prog_type type)
{
        switch (type) {
        case BPF_PROG_TYPE_SOCKET_FILTER:
        case BPF_PROG_TYPE_SCHED_CLS:
        case BPF_PROG_TYPE_SCHED_ACT:
                return true;
        default:
                return false;
        }
}

/* verify safety of LD_ABS|LD_IND instructions:
 * - they can only appear in the programs where ctx == skb
 * - since they are wrappers of function calls, they scratch R1-R5 registers,
 *   preserve R6-R9, and store return value into R0
 *
 * Implicit input:
 *   ctx == skb == R6 == CTX
 *
 * Explicit input:
 *   SRC == any register
 *   IMM == 32-bit immediate
 *
 * Output:
 *   R0 - 8/16/32-bit skb data converted to cpu endianness
 */
static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
{
        struct bpf_reg_state *regs = cur_regs(env);
        static const int ctx_reg = BPF_REG_6;
        u8 mode = BPF_MODE(insn->code);
        int i, err;

        if (!may_access_skb(resolve_prog_type(env->prog))) {
                verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
                return -EINVAL;
        }

        if (!env->ops->gen_ld_abs) {
                verbose(env, "bpf verifier is misconfigured\n");
                return -EINVAL;
        }

        if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
            BPF_SIZE(insn->code) == BPF_DW ||
            (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
                verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
                return -EINVAL;
        }

        /* check whether implicit source operand (register R6) is readable */
        err = check_reg_arg(env, ctx_reg, SRC_OP);
        if (err)
                return err;

        /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
         * gen_ld_abs() may terminate the program at runtime, leading to
         * reference leak.
         */
        err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]");
        if (err)
                return err;

        if (regs[ctx_reg].type != PTR_TO_CTX) {
                verbose(env,
                        "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
                return -EINVAL;
        }

        if (mode == BPF_IND) {
                /* check explicit source operand */
                err = check_reg_arg(env, insn->src_reg, SRC_OP);
                if (err)
                        return err;
        }

        err = check_ptr_off_reg(env, &regs[ctx_reg], ctx_reg);
        if (err < 0)
                return err;

        /* reset caller saved regs to unreadable */
        for (i = 0; i < CALLER_SAVED_REGS; i++) {
                mark_reg_not_init(env, regs, caller_saved[i]);
                check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
        }

        /* mark destination R0 register as readable, since it contains
         * the value fetched from the packet.
         * Already marked as written above.
         */
        mark_reg_unknown(env, regs, BPF_REG_0);
        /* ld_abs load up to 32-bit skb data. */
        regs[BPF_REG_0].subreg_def = env->insn_idx + 1;
        return 0;
}

static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name)
{
        const char *exit_ctx = "At program exit";
        struct tnum enforce_attach_type_range = tnum_unknown;
        const struct bpf_prog *prog = env->prog;
        struct bpf_reg_state *reg;
        struct bpf_retval_range range = retval_range(0, 1);
        enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
        int err;
        struct bpf_func_state *frame = env->cur_state->frame[0];
        const bool is_subprog = frame->subprogno;
        bool return_32bit = false;

        /* LSM and struct_ops func-ptr's return type could be "void" */
        if (!is_subprog || frame->in_exception_callback_fn) {
                switch (prog_type) {
                case BPF_PROG_TYPE_LSM:
                        if (prog->expected_attach_type == BPF_LSM_CGROUP)
                                /* See below, can be 0 or 0-1 depending on hook. */
                                break;
                        fallthrough;
                case BPF_PROG_TYPE_STRUCT_OPS:
                        if (!prog->aux->attach_func_proto->type)
                                return 0;
                        break;
                default:
                        break;
                }
        }

        /* eBPF calling convention is such that R0 is used
         * to return the value from eBPF program.
         * Make sure that it's readable at this time
         * of bpf_exit, which means that program wrote
         * something into it earlier
         */
        err = check_reg_arg(env, regno, SRC_OP);
        if (err)
                return err;

        if (is_pointer_value(env, regno)) {
                verbose(env, "R%d leaks addr as return value\n", regno);
                return -EACCES;
        }

        reg = cur_regs(env) + regno;

        if (frame->in_async_callback_fn) {
                /* enforce return zero from async callbacks like timer */
                exit_ctx = "At async callback return";
                range = retval_range(0, 0);
                goto enforce_retval;
        }

        if (is_subprog && !frame->in_exception_callback_fn) {
                if (reg->type != SCALAR_VALUE) {
                        verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n",
                                regno, reg_type_str(env, reg->type));
                        return -EINVAL;
                }
                return 0;
        }

        switch (prog_type) {
        case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
                if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
                    env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
                    env->prog->expected_attach_type == BPF_CGROUP_UNIX_RECVMSG ||
                    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
                    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
                    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETPEERNAME ||
                    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
                    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME ||
                    env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME)
                        range = retval_range(1, 1);
                if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
                    env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
                        range = retval_range(0, 3);
                break;
        case BPF_PROG_TYPE_CGROUP_SKB:
                if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
                        range = retval_range(0, 3);
                        enforce_attach_type_range = tnum_range(2, 3);
                }
                break;
        case BPF_PROG_TYPE_CGROUP_SOCK:
        case BPF_PROG_TYPE_SOCK_OPS:
        case BPF_PROG_TYPE_CGROUP_DEVICE:
        case BPF_PROG_TYPE_CGROUP_SYSCTL:
        case BPF_PROG_TYPE_CGROUP_SOCKOPT:
                break;
        case BPF_PROG_TYPE_RAW_TRACEPOINT:
                if (!env->prog->aux->attach_btf_id)
                        return 0;
                range = retval_range(0, 0);
                break;
        case BPF_PROG_TYPE_TRACING:
                switch (env->prog->expected_attach_type) {
                case BPF_TRACE_FENTRY:
                case BPF_TRACE_FEXIT:
                        range = retval_range(0, 0);
                        break;
                case BPF_TRACE_RAW_TP:
                case BPF_MODIFY_RETURN:
                        return 0;
                case BPF_TRACE_ITER:
                        break;
                default:
                        return -ENOTSUPP;
                }
                break;
        case BPF_PROG_TYPE_KPROBE:
                switch (env->prog->expected_attach_type) {
                case BPF_TRACE_KPROBE_SESSION:
                case BPF_TRACE_UPROBE_SESSION:
                        range = retval_range(0, 1);
                        break;
                default:
                        return 0;
                }
                break;
        case BPF_PROG_TYPE_SK_LOOKUP:
                range = retval_range(SK_DROP, SK_PASS);
                break;

        case BPF_PROG_TYPE_LSM:
                if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {
                        /* no range found, any return value is allowed */
                        if (!get_func_retval_range(env->prog, &range))
                                return 0;
                        /* no restricted range, any return value is allowed */
                        if (range.minval == S32_MIN && range.maxval == S32_MAX)
                                return 0;
                        return_32bit = true;
                } else if (!env->prog->aux->attach_func_proto->type) {
                        /* Make sure programs that attach to void
                         * hooks don't try to modify return value.
                         */
                        range = retval_range(1, 1);
                }
                break;

        case BPF_PROG_TYPE_NETFILTER:
                range = retval_range(NF_DROP, NF_ACCEPT);
                break;
        case BPF_PROG_TYPE_EXT:
                /* freplace program can return anything as its return value
                 * depends on the to-be-replaced kernel func or bpf program.
                 */
        default:
                return 0;
        }

enforce_retval:
        if (reg->type != SCALAR_VALUE) {
                verbose(env, "%s the register R%d is not a known value (%s)\n",
                        exit_ctx, regno, reg_type_str(env, reg->type));
                return -EINVAL;
        }

        err = mark_chain_precision(env, regno);
        if (err)
                return err;

        if (!retval_range_within(range, reg, return_32bit)) {
                verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name);
                if (!is_subprog &&
                    prog->expected_attach_type == BPF_LSM_CGROUP &&
                    prog_type == BPF_PROG_TYPE_LSM &&
                    !prog->aux->attach_func_proto->type)
                        verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
                return -EINVAL;
        }

        if (!tnum_is_unknown(enforce_attach_type_range) &&
            tnum_in(enforce_attach_type_range, reg->var_off))
                env->prog->enforce_expected_attach_type = 1;
        return 0;
}

static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off)
{
        struct bpf_subprog_info *subprog;

        subprog = find_containing_subprog(env, off);
        subprog->changes_pkt_data = true;
}

/* 't' is an index of a call-site.
 * 'w' is a callee entry point.
 * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
 * Rely on DFS traversal order and absence of recursive calls to guarantee that
 * callee's change_pkt_data marks would be correct at that moment.
 */
static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
{
        struct bpf_subprog_info *caller, *callee;

        caller = find_containing_subprog(env, t);
        callee = find_containing_subprog(env, w);
        caller->changes_pkt_data |= callee->changes_pkt_data;
}

/* non-recursive DFS pseudo code
 * 1  procedure DFS-iterative(G,v):
 * 2      label v as discovered
 * 3      let S be a stack
 * 4      S.push(v)
 * 5      while S is not empty
 * 6            t <- S.peek()
 * 7            if t is what we're looking for:
 * 8                return t
 * 9            for all edges e in G.adjacentEdges(t) do
 * 10               if edge e is already labelled
 * 11                   continue with the next edge
 * 12               w <- G.adjacentVertex(t,e)
 * 13               if vertex w is not discovered and not explored
 * 14                   label e as tree-edge
 * 15                   label w as discovered
 * 16                   S.push(w)
 * 17                   continue at 5
 * 18               else if vertex w is discovered
 * 19                   label e as back-edge
 * 20               else
 * 21                   // vertex w is explored
 * 22                   label e as forward- or cross-edge
 * 23           label t as explored
 * 24           S.pop()
 *
 * convention:
 * 0x10 - discovered
 * 0x11 - discovered and fall-through edge labelled
 * 0x12 - discovered and fall-through and branch edges labelled
 * 0x20 - explored
 */

enum {
        DISCOVERED = 0x10,
        EXPLORED = 0x20,
        FALLTHROUGH = 1,
        BRANCH = 2,
};

static void mark_prune_point(struct bpf_verifier_env *env, int idx)
{
        env->insn_aux_data[idx].prune_point = true;
}

static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx)
{
        return env->insn_aux_data[insn_idx].prune_point;
}

static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)
{
        env->insn_aux_data[idx].force_checkpoint = true;
}

static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)
{
        return env->insn_aux_data[insn_idx].force_checkpoint;
}

static void mark_calls_callback(struct bpf_verifier_env *env, int idx)
{
        env->insn_aux_data[idx].calls_callback = true;
}

static bool calls_callback(struct bpf_verifier_env *env, int insn_idx)
{
        return env->insn_aux_data[insn_idx].calls_callback;
}

enum {
        DONE_EXPLORING = 0,
        KEEP_EXPLORING = 1,
};

/* t, w, e - match pseudo-code above:
 * t - index of current instruction
 * w - next instruction
 * e - edge
 */
static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
{
        int *insn_stack = env->cfg.insn_stack;
        int *insn_state = env->cfg.insn_state;

        if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
                return DONE_EXPLORING;

        if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
                return DONE_EXPLORING;

        if (w < 0 || w >= env->prog->len) {
                verbose_linfo(env, t, "%d: ", t);
                verbose(env, "jump out of range from insn %d to %d\n", t, w);
                return -EINVAL;
        }

        if (e == BRANCH) {
                /* mark branch target for state pruning */
                mark_prune_point(env, w);
                mark_jmp_point(env, w);
        }

        if (insn_state[w] == 0) {
                /* tree-edge */
                insn_state[t] = DISCOVERED | e;
                insn_state[w] = DISCOVERED;
                if (env->cfg.cur_stack >= env->prog->len)
                        return -E2BIG;
                insn_stack[env->cfg.cur_stack++] = w;
                return KEEP_EXPLORING;
        } else if ((insn_state[w] & 0xF0) == DISCOVERED) {
                if (env->bpf_capable)
                        return DONE_EXPLORING;
                verbose_linfo(env, t, "%d: ", t);
                verbose_linfo(env, w, "%d: ", w);
                verbose(env, "back-edge from insn %d to %d\n", t, w);
                return -EINVAL;
        } else if (insn_state[w] == EXPLORED) {
                /* forward- or cross-edge */
                insn_state[t] = DISCOVERED | e;
        } else {
                verbose(env, "insn state internal bug\n");
                return -EFAULT;
        }
        return DONE_EXPLORING;
}

static int visit_func_call_insn(int t, struct bpf_insn *insns,
                                struct bpf_verifier_env *env,
                                bool visit_callee)
{
        int ret, insn_sz;
        int w;

        insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1;
        ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);
        if (ret)
                return ret;

        mark_prune_point(env, t + insn_sz);
        /* when we exit from subprog, we need to record non-linear history */
        mark_jmp_point(env, t + insn_sz);

        if (visit_callee) {
                w = t + insns[t].imm + 1;
                mark_prune_point(env, t);
                merge_callee_effects(env, t, w);
                ret = push_insn(t, w, BRANCH, env);
        }
        return ret;
}

/* Bitmask with 1s for all caller saved registers */
#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)

/* Return a bitmask specifying which caller saved registers are
 * clobbered by a call to a helper *as if* this helper follows
 * bpf_fastcall contract:
 * - includes R0 if function is non-void;
 * - includes R1-R5 if corresponding parameter has is described
 *   in the function prototype.
 */
static u32 helper_fastcall_clobber_mask(const struct bpf_func_proto *fn)
{
        u32 mask;
        int i;

        mask = 0;
        if (fn->ret_type != RET_VOID)
                mask |= BIT(BPF_REG_0);
        for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i)
                if (fn->arg_type[i] != ARG_DONTCARE)
                        mask |= BIT(BPF_REG_1 + i);
        return mask;
}

/* True if do_misc_fixups() replaces calls to helper number 'imm',
 * replacement patch is presumed to follow bpf_fastcall contract
 * (see mark_fastcall_pattern_for_call() below).
 */
static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
{
        switch (imm) {
#ifdef CONFIG_X86_64
        case BPF_FUNC_get_smp_processor_id:
                return env->prog->jit_requested && bpf_jit_supports_percpu_insn();
#endif
        default:
                return false;
        }
}

/* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */
static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)
{
        u32 vlen, i, mask;

        vlen = btf_type_vlen(meta->func_proto);
        mask = 0;
        if (!btf_type_is_void(btf_type_by_id(meta->btf, meta->func_proto->type)))
                mask |= BIT(BPF_REG_0);
        for (i = 0; i < vlen; ++i)
                mask |= BIT(BPF_REG_1 + i);
        return mask;
}

/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */
static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)
{
        return meta->kfunc_flags & KF_FASTCALL;
}

/* LLVM define a bpf_fastcall function attribute.
 * This attribute means that function scratches only some of
 * the caller saved registers defined by ABI.
 * For BPF the set of such registers could be defined as follows:
 * - R0 is scratched only if function is non-void;
 * - R1-R5 are scratched only if corresponding parameter type is defined
 *   in the function prototype.
 *
 * The contract between kernel and clang allows to simultaneously use
 * such functions and maintain backwards compatibility with old
 * kernels that don't understand bpf_fastcall calls:
 *
 * - for bpf_fastcall calls clang allocates registers as-if relevant r0-r5
 *   registers are not scratched by the call;
 *
 * - as a post-processing step, clang visits each bpf_fastcall call and adds
 *   spill/fill for every live r0-r5;
 *
 * - stack offsets used for the spill/fill are allocated as lowest
 *   stack offsets in whole function and are not used for any other
 *   purposes;
 *
 * - when kernel loads a program, it looks for such patterns
 *   (bpf_fastcall function surrounded by spills/fills) and checks if
 *   spill/fill stack offsets are used exclusively in fastcall patterns;
 *
 * - if so, and if verifier or current JIT inlines the call to the
 *   bpf_fastcall function (e.g. a helper call), kernel removes unnecessary
 *   spill/fill pairs;
 *
 * - when old kernel loads a program, presence of spill/fill pairs
 *   keeps BPF program valid, albeit slightly less efficient.
 *
 * For example:
 *
 *   r1 = 1;
 *   r2 = 2;
 *   *(u64 *)(r10 - 8)  = r1;            r1 = 1;
 *   *(u64 *)(r10 - 16) = r2;            r2 = 2;
 *   call %[to_be_inlined]         -->   call %[to_be_inlined]
 *   r2 = *(u64 *)(r10 - 16);            r0 = r1;
 *   r1 = *(u64 *)(r10 - 8);             r0 += r2;
 *   r0 = r1;                            exit;
 *   r0 += r2;
 *   exit;
 *
 * The purpose of mark_fastcall_pattern_for_call is to:
 * - look for such patterns;
 * - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern;
 * - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction;
 * - update env->subprog_info[*]->fastcall_stack_off to find an offset
 *   at which bpf_fastcall spill/fill stack slots start;
 * - update env->subprog_info[*]->keep_fastcall_stack.
 *
 * The .fastcall_pattern and .fastcall_stack_off are used by
 * check_fastcall_stack_contract() to check if every stack access to
 * fastcall spill/fill stack slot originates from spill/fill
 * instructions, members of fastcall patterns.
 *
 * If such condition holds true for a subprogram, fastcall patterns could
 * be rewritten by remove_fastcall_spills_fills().
 * Otherwise bpf_fastcall patterns are not changed in the subprogram
 * (code, presumably, generated by an older clang version).
 *
 * For example, it is *not* safe to remove spill/fill below:
 *
 *   r1 = 1;
 *   *(u64 *)(r10 - 8)  = r1;            r1 = 1;
 *   call %[to_be_inlined]         -->   call %[to_be_inlined]
 *   r1 = *(u64 *)(r10 - 8);             r0 = *(u64 *)(r10 - 8);  <---- wrong !!!
 *   r0 = *(u64 *)(r10 - 8);             r0 += r1;
 *   r0 += r1;                           exit;
 *   exit;
 */
static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
                                           struct bpf_subprog_info *subprog,
                                           int insn_idx, s16 lowest_off)
{
        struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx;
        struct bpf_insn *call = &env->prog->insnsi[insn_idx];
        const struct bpf_func_proto *fn;
        u32 clobbered_regs_mask = ALL_CALLER_SAVED_REGS;
        u32 expected_regs_mask;
        bool can_be_inlined = false;
        s16 off;
        int i;

        if (bpf_helper_call(call)) {
                if (get_helper_proto(env, call->imm, &fn) < 0)
                        /* error would be reported later */
                        return;
                clobbered_regs_mask = helper_fastcall_clobber_mask(fn);
                can_be_inlined = fn->allow_fastcall &&
                                 (verifier_inlines_helper_call(env, call->imm) ||
                                  bpf_jit_inlines_helper_call(call->imm));
        }

        if (bpf_pseudo_kfunc_call(call)) {
                struct bpf_kfunc_call_arg_meta meta;
                int err;

                err = fetch_kfunc_meta(env, call, &meta, NULL);
                if (err < 0)
                        /* error would be reported later */
                        return;

                clobbered_regs_mask = kfunc_fastcall_clobber_mask(&meta);
                can_be_inlined = is_fastcall_kfunc_call(&meta);
        }

        if (clobbered_regs_mask == ALL_CALLER_SAVED_REGS)
                return;

        /* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */
        expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS;

        /* match pairs of form:
         *
         * *(u64 *)(r10 - Y) = rX   (where Y % 8 == 0)
         * ...
         * call %[to_be_inlined]
         * ...
         * rX = *(u64 *)(r10 - Y)
         */
        for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) {
                if (insn_idx - i < 0 || insn_idx + i >= env->prog->len)
                        break;
                stx = &insns[insn_idx - i];
                ldx = &insns[insn_idx + i];
                /* must be a stack spill/fill pair */
                if (stx->code != (BPF_STX | BPF_MEM | BPF_DW) ||
                    ldx->code != (BPF_LDX | BPF_MEM | BPF_DW) ||
                    stx->dst_reg != BPF_REG_10 ||
                    ldx->src_reg != BPF_REG_10)
                        break;
                /* must be a spill/fill for the same reg */
                if (stx->src_reg != ldx->dst_reg)
                        break;
                /* must be one of the previously unseen registers */
                if ((BIT(stx->src_reg) & expected_regs_mask) == 0)
                        break;
                /* must be a spill/fill for the same expected offset,
                 * no need to check offset alignment, BPF_DW stack access
                 * is always 8-byte aligned.
                 */
                if (stx->off != off || ldx->off != off)
                        break;
                expected_regs_mask &= ~BIT(stx->src_reg);
                env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;
                env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;
        }
        if (i == 1)
                return;

        /* Conditionally set 'fastcall_spills_num' to allow forward
         * compatibility when more helper functions are marked as
         * bpf_fastcall at compile time than current kernel supports, e.g:
         *
         *   1: *(u64 *)(r10 - 8) = r1
         *   2: call A                  ;; assume A is bpf_fastcall for current kernel
         *   3: r1 = *(u64 *)(r10 - 8)
         *   4: *(u64 *)(r10 - 8) = r1
         *   5: call B                  ;; assume B is not bpf_fastcall for current kernel
         *   6: r1 = *(u64 *)(r10 - 8)
         *
         * There is no need to block bpf_fastcall rewrite for such program.
         * Set 'fastcall_pattern' for both calls to keep check_fastcall_stack_contract() happy,
         * don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()
         * does not remove spill/fill pair {4,6}.
         */
        if (can_be_inlined)
                env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;
        else
                subprog->keep_fastcall_stack = 1;
        subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);
}

static int mark_fastcall_patterns(struct bpf_verifier_env *env)
{
        struct bpf_subprog_info *subprog = env->subprog_info;
        struct bpf_insn *insn;
        s16 lowest_off;
        int s, i;

        for (s = 0; s < env->subprog_cnt; ++s, ++subprog) {
                /* find lowest stack spill offset used in this subprog */
                lowest_off = 0;
                for (i = subprog->start; i < (subprog + 1)->start; ++i) {
                        insn = env->prog->insnsi + i;
                        if (insn->code != (BPF_STX | BPF_MEM | BPF_DW) ||
                            insn->dst_reg != BPF_REG_10)
                                continue;
                        lowest_off = min(lowest_off, insn->off);
                }
                /* use this offset to find fastcall patterns */
                for (i = subprog->start; i < (subprog + 1)->start; ++i) {
                        insn = env->prog->insnsi + i;
                        if (insn->code != (BPF_JMP | BPF_CALL))
                                continue;
                        mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);
                }
        }
        return 0;
}

/* Visits the instruction at index t and returns one of the following:
 *  < 0 - an error occurred
 *  DONE_EXPLORING - the instruction was fully explored
 *  KEEP_EXPLORING - there is still work to be done before it is fully explored
 */
static int visit_insn(int t, struct bpf_verifier_env *env)
{
        struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];
        int ret, off, insn_sz;

        if (bpf_pseudo_func(insn))
                return visit_func_call_insn(t, insns, env, true);

        /* All non-branch instructions have a single fall-through edge. */
        if (BPF_CLASS(insn->code) != BPF_JMP &&
            BPF_CLASS(insn->code) != BPF_JMP32) {
                insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
                return push_insn(t, t + insn_sz, FALLTHROUGH, env);
        }

        switch (BPF_OP(insn->code)) {
        case BPF_EXIT:
                return DONE_EXPLORING;

        case BPF_CALL:
                if (is_async_callback_calling_insn(insn))
                        /* Mark this call insn as a prune point to trigger
                         * is_state_visited() check before call itself is
                         * processed by __check_func_call(). Otherwise new
                         * async state will be pushed for further exploration.
                         */
                        mark_prune_point(env, t);
                /* For functions that invoke callbacks it is not known how many times
                 * callback would be called. Verifier models callback calling functions
                 * by repeatedly visiting callback bodies and returning to origin call
                 * instruction.
                 * In order to stop such iteration verifier needs to identify when a
                 * state identical some state from a previous iteration is reached.
                 * Check below forces creation of checkpoint before callback calling
                 * instruction to allow search for such identical states.
                 */
                if (is_sync_callback_calling_insn(insn)) {
                        mark_calls_callback(env, t);
                        mark_force_checkpoint(env, t);
                        mark_prune_point(env, t);
                        mark_jmp_point(env, t);
                }
                if (bpf_helper_call(insn) && bpf_helper_changes_pkt_data(insn->imm))
                        mark_subprog_changes_pkt_data(env, t);
                if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
                        struct bpf_kfunc_call_arg_meta meta;

                        ret = fetch_kfunc_meta(env, insn, &meta, NULL);
                        if (ret == 0 && is_iter_next_kfunc(&meta)) {
                                mark_prune_point(env, t);
                                /* Checking and saving state checkpoints at iter_next() call
                                 * is crucial for fast convergence of open-coded iterator loop
                                 * logic, so we need to force it. If we don't do that,
                                 * is_state_visited() might skip saving a checkpoint, causing
                                 * unnecessarily long sequence of not checkpointed
                                 * instructions and jumps, leading to exhaustion of jump
                                 * history buffer, and potentially other undesired outcomes.
                                 * It is expected that with correct open-coded iterators
                                 * convergence will happen quickly, so we don't run a risk of
                                 * exhausting memory.
                                 */
                                mark_force_checkpoint(env, t);
                        }
                }
                return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);

        case BPF_JA:
                if (BPF_SRC(insn->code) != BPF_K)
                        return -EINVAL;

                if (BPF_CLASS(insn->code) == BPF_JMP)
                        off = insn->off;
                else
                        off = insn->imm;

                /* unconditional jump with single edge */
                ret = push_insn(t, t + off + 1, FALLTHROUGH, env);
                if (ret)
                        return ret;

                mark_prune_point(env, t + off + 1);
                mark_jmp_point(env, t + off + 1);

                return ret;

        default:
                /* conditional jump with two edges */
                mark_prune_point(env, t);
                if (is_may_goto_insn(insn))
                        mark_force_checkpoint(env, t);

                ret = push_insn(t, t + 1, FALLTHROUGH, env);
                if (ret)
                        return ret;

                return push_insn(t, t + insn->off + 1, BRANCH, env);
        }
}

/* non-recursive depth-first-search to detect loops in BPF program
 * loop == back-edge in directed graph
 */
static int check_cfg(struct bpf_verifier_env *env)
{
        int insn_cnt = env->prog->len;
        int *insn_stack, *insn_state;
        int ex_insn_beg, i, ret = 0;
        bool ex_done = false;

        insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
        if (!insn_state)
                return -ENOMEM;

        insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
        if (!insn_stack) {
                kvfree(insn_state);
                return -ENOMEM;
        }

        insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
        insn_stack[0] = 0; /* 0 is the first instruction */
        env->cfg.cur_stack = 1;

walk_cfg:
        while (env->cfg.cur_stack > 0) {
                int t = insn_stack[env->cfg.cur_stack - 1];

                ret = visit_insn(t, env);
                switch (ret) {
                case DONE_EXPLORING:
                        insn_state[t] = EXPLORED;
                        env->cfg.cur_stack--;
                        break;
                case KEEP_EXPLORING:
                        break;
                default:
                        if (ret > 0) {
                                verbose(env, "visit_insn internal bug\n");
                                ret = -EFAULT;
                        }
                        goto err_free;
                }
        }

        if (env->cfg.cur_stack < 0) {
                verbose(env, "pop stack internal bug\n");
                ret = -EFAULT;
                goto err_free;
        }

        if (env->exception_callback_subprog && !ex_done) {
                ex_insn_beg = env->subprog_info[env->exception_callback_subprog].start;

                insn_state[ex_insn_beg] = DISCOVERED;
                insn_stack[0] = ex_insn_beg;
                env->cfg.cur_stack = 1;
                ex_done = true;
                goto walk_cfg;
        }

        for (i = 0; i < insn_cnt; i++) {
                struct bpf_insn *insn = &env->prog->insnsi[i];

                if (insn_state[i] != EXPLORED) {
                        verbose(env, "unreachable insn %d\n", i);
                        ret = -EINVAL;
                        goto err_free;
                }
                if (bpf_is_ldimm64(insn)) {
                        if (insn_state[i + 1] != 0) {
                                verbose(env, "jump into the middle of ldimm64 insn %d\n", i);
                                ret = -EINVAL;
                                goto err_free;
                        }
                        i++; /* skip second half of ldimm64 */
                }
        }
        ret = 0; /* cfg looks good */
        env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;

err_free:
        kvfree(insn_state);
        kvfree(insn_stack);
        env->cfg.insn_state = env->cfg.insn_stack = NULL;
        return ret;
}

static int check_abnormal_return(struct bpf_verifier_env *env)
{
        int i;

        for (i = 1; i < env->subprog_cnt; i++) {
                if (env->subprog_info[i].has_ld_abs) {
                        verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
                        return -EINVAL;
                }
                if (env->subprog_info[i].has_tail_call) {
                        verbose(env, "tail_call is not allowed in subprogs without BTF\n");
                        return -EINVAL;
                }
        }
        return 0;
}

/* The minimum supported BTF func info size */
#define MIN_BPF_FUNCINFO_SIZE        8
#define MAX_FUNCINFO_REC_SIZE        252

static int check_btf_func_early(struct bpf_verifier_env *env,
                                const union bpf_attr *attr,
                                bpfptr_t uattr)
{
        u32 krec_size = sizeof(struct bpf_func_info);
        const struct btf_type *type, *func_proto;
        u32 i, nfuncs, urec_size, min_size;
        struct bpf_func_info *krecord;
        struct bpf_prog *prog;
        const struct btf *btf;
        u32 prev_offset = 0;
        bpfptr_t urecord;
        int ret = -ENOMEM;

        nfuncs = attr->func_info_cnt;
        if (!nfuncs) {
                if (check_abnormal_return(env))
                        return -EINVAL;
                return 0;
        }

        urec_size = attr->func_info_rec_size;
        if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
            urec_size > MAX_FUNCINFO_REC_SIZE ||
            urec_size % sizeof(u32)) {
                verbose(env, "invalid func info rec size %u\n", urec_size);
                return -EINVAL;
        }

        prog = env->prog;
        btf = prog->aux->btf;

        urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
        min_size = min_t(u32, krec_size, urec_size);

        krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
        if (!krecord)
                return -ENOMEM;

        for (i = 0; i < nfuncs; i++) {
                ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
                if (ret) {
                        if (ret == -E2BIG) {
                                verbose(env, "nonzero tailing record in func info");
                                /* set the size kernel expects so loader can zero
                                 * out the rest of the record.
                                 */
                                if (copy_to_bpfptr_offset(uattr,
                                                          offsetof(union bpf_attr, func_info_rec_size),
                                                          &min_size, sizeof(min_size)))
                                        ret = -EFAULT;
                        }
                        goto err_free;
                }

                if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {
                        ret = -EFAULT;
                        goto err_free;
                }

                /* check insn_off */
                ret = -EINVAL;
                if (i == 0) {
                        if (krecord[i].insn_off) {
                                verbose(env,
                                        "nonzero insn_off %u for the first func info record",
                                        krecord[i].insn_off);
                                goto err_free;
                        }
                } else if (krecord[i].insn_off <= prev_offset) {
                        verbose(env,
                                "same or smaller insn offset (%u) than previous func info record (%u)",
                                krecord[i].insn_off, prev_offset);
                        goto err_free;
                }

                /* check type_id */
                type = btf_type_by_id(btf, krecord[i].type_id);
                if (!type || !btf_type_is_func(type)) {
                        verbose(env, "invalid type id %d in func info",
                                krecord[i].type_id);
                        goto err_free;
                }

                func_proto = btf_type_by_id(btf, type->type);
                if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))
                        /* btf_func_check() already verified it during BTF load */
                        goto err_free;

                prev_offset = krecord[i].insn_off;
                bpfptr_add(&urecord, urec_size);
        }

        prog->aux->func_info = krecord;
        prog->aux->func_info_cnt = nfuncs;
        return 0;

err_free:
        kvfree(krecord);
        return ret;
}

static int check_btf_func(struct bpf_verifier_env *env,
                          const union bpf_attr *attr,
                          bpfptr_t uattr)
{
        const struct btf_type *type, *func_proto, *ret_type;
        u32 i, nfuncs, urec_size;
        struct bpf_func_info *krecord;
        struct bpf_func_info_aux *info_aux = NULL;
        struct bpf_prog *prog;
        const struct btf *btf;
        bpfptr_t urecord;
        bool scalar_return;
        int ret = -ENOMEM;

        nfuncs = attr->func_info_cnt;
        if (!nfuncs) {
                if (check_abnormal_return(env))
                        return -EINVAL;
                return 0;
        }
        if (nfuncs != env->subprog_cnt) {
                verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
                return -EINVAL;
        }

        urec_size = attr->func_info_rec_size;

        prog = env->prog;
        btf = prog->aux->btf;

        urecord = make_bpfptr(attr->func_info, uattr.is_kernel);

        krecord = prog->aux->func_info;
        info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN);
        if (!info_aux)
                return -ENOMEM;

        for (i = 0; i < nfuncs; i++) {
                /* check insn_off */
                ret = -EINVAL;

                if (env->subprog_info[i].start != krecord[i].insn_off) {
                        verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
                        goto err_free;
                }

                /* Already checked type_id */
                type = btf_type_by_id(btf, krecord[i].type_id);
                info_aux[i].linkage = BTF_INFO_VLEN(type->info);
                /* Already checked func_proto */
                func_proto = btf_type_by_id(btf, type->type);

                ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
                scalar_return =
                        btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type);
                if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
                        verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
                        goto err_free;
                }
                if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
                        verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
                        goto err_free;
                }

                bpfptr_add(&urecord, urec_size);
        }

        prog->aux->func_info_aux = info_aux;
        return 0;

err_free:
        kfree(info_aux);
        return ret;
}

static void adjust_btf_func(struct bpf_verifier_env *env)
{
        struct bpf_prog_aux *aux = env->prog->aux;
        int i;

        if (!aux->func_info)
                return;

        /* func_info is not available for hidden subprogs */
        for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++)
                aux->func_info[i].insn_off = env->subprog_info[i].start;
}

#define MIN_BPF_LINEINFO_SIZE        offsetofend(struct bpf_line_info, line_col)
#define MAX_LINEINFO_REC_SIZE        MAX_FUNCINFO_REC_SIZE

static int check_btf_line(struct bpf_verifier_env *env,
                          const union bpf_attr *attr,
                          bpfptr_t uattr)
{
        u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
        struct bpf_subprog_info *sub;
        struct bpf_line_info *linfo;
        struct bpf_prog *prog;
        const struct btf *btf;
        bpfptr_t ulinfo;
        int err;

        nr_linfo = attr->line_info_cnt;
        if (!nr_linfo)
                return 0;
        if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info))
                return -EINVAL;

        rec_size = attr->line_info_rec_size;
        if (rec_size < MIN_BPF_LINEINFO_SIZE ||
            rec_size > MAX_LINEINFO_REC_SIZE ||
            rec_size & (sizeof(u32) - 1))
                return -EINVAL;

        /* Need to zero it in case the userspace may
         * pass in a smaller bpf_line_info object.
         */
        linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
                         GFP_KERNEL | __GFP_NOWARN);
        if (!linfo)
                return -ENOMEM;

        prog = env->prog;
        btf = prog->aux->btf;

        s = 0;
        sub = env->subprog_info;
        ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);
        expected_size = sizeof(struct bpf_line_info);
        ncopy = min_t(u32, expected_size, rec_size);
        for (i = 0; i < nr_linfo; i++) {
                err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
                if (err) {
                        if (err == -E2BIG) {
                                verbose(env, "nonzero tailing record in line_info");
                                if (copy_to_bpfptr_offset(uattr,
                                                          offsetof(union bpf_attr, line_info_rec_size),
                                                          &expected_size, sizeof(expected_size)))
                                        err = -EFAULT;
                        }
                        goto err_free;
                }

                if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {
                        err = -EFAULT;
                        goto err_free;
                }

                /*
                 * Check insn_off to ensure
                 * 1) strictly increasing AND
                 * 2) bounded by prog->len
                 *
                 * The linfo[0].insn_off == 0 check logically falls into
                 * the later "missing bpf_line_info for func..." case
                 * because the first linfo[0].insn_off must be the
                 * first sub also and the first sub must have
                 * subprog_info[0].start == 0.
                 */
                if ((i && linfo[i].insn_off <= prev_offset) ||
                    linfo[i].insn_off >= prog->len) {
                        verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
                                i, linfo[i].insn_off, prev_offset,
                                prog->len);
                        err = -EINVAL;
                        goto err_free;
                }

                if (!prog->insnsi[linfo[i].insn_off].code) {
                        verbose(env,
                                "Invalid insn code at line_info[%u].insn_off\n",
                                i);
                        err = -EINVAL;
                        goto err_free;
                }

                if (!btf_name_by_offset(btf, linfo[i].line_off) ||
                    !btf_name_by_offset(btf, linfo[i].file_name_off)) {
                        verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
                        err = -EINVAL;
                        goto err_free;
                }

                if (s != env->subprog_cnt) {
                        if (linfo[i].insn_off == sub[s].start) {
                                sub[s].linfo_idx = i;
                                s++;
                        } else if (sub[s].start < linfo[i].insn_off) {
                                verbose(env, "missing bpf_line_info for func#%u\n", s);
                                err = -EINVAL;
                                goto err_free;
                        }
                }

                prev_offset = linfo[i].insn_off;
                bpfptr_add(&ulinfo, rec_size);
        }

        if (s != env->subprog_cnt) {
                verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
                        env->subprog_cnt - s, s);
                err = -EINVAL;
                goto err_free;
        }

        prog->aux->linfo = linfo;
        prog->aux->nr_linfo = nr_linfo;

        return 0;

err_free:
        kvfree(linfo);
        return err;
}

#define MIN_CORE_RELO_SIZE        sizeof(struct bpf_core_relo)
#define MAX_CORE_RELO_SIZE        MAX_FUNCINFO_REC_SIZE

static int check_core_relo(struct bpf_verifier_env *env,
                           const union bpf_attr *attr,
                           bpfptr_t uattr)
{
        u32 i, nr_core_relo, ncopy, expected_size, rec_size;
        struct bpf_core_relo core_relo = {};
        struct bpf_prog *prog = env->prog;
        const struct btf *btf = prog->aux->btf;
        struct bpf_core_ctx ctx = {
                .log = &env->log,
                .btf = btf,
        };
        bpfptr_t u_core_relo;
        int err;

        nr_core_relo = attr->core_relo_cnt;
        if (!nr_core_relo)
                return 0;
        if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))
                return -EINVAL;

        rec_size = attr->core_relo_rec_size;
        if (rec_size < MIN_CORE_RELO_SIZE ||
            rec_size > MAX_CORE_RELO_SIZE ||
            rec_size % sizeof(u32))
                return -EINVAL;

        u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);
        expected_size = sizeof(struct bpf_core_relo);
        ncopy = min_t(u32, expected_size, rec_size);

        /* Unlike func_info and line_info, copy and apply each CO-RE
         * relocation record one at a time.
         */
        for (i = 0; i < nr_core_relo; i++) {
                /* future proofing when sizeof(bpf_core_relo) changes */
                err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);
                if (err) {
                        if (err == -E2BIG) {
                                verbose(env, "nonzero tailing record in core_relo");
                                if (copy_to_bpfptr_offset(uattr,
                                                          offsetof(union bpf_attr, core_relo_rec_size),
                                                          &expected_size, sizeof(expected_size)))
                                        err = -EFAULT;
                        }
                        break;
                }

                if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {
                        err = -EFAULT;
                        break;
                }

                if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {
                        verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",
                                i, core_relo.insn_off, prog->len);
                        err = -EINVAL;
                        break;
                }

                err = bpf_core_apply(&ctx, &core_relo, i,
                                     &prog->insnsi[core_relo.insn_off / 8]);
                if (err)
                        break;
                bpfptr_add(&u_core_relo, rec_size);
        }
        return err;
}

static int check_btf_info_early(struct bpf_verifier_env *env,
                                const union bpf_attr *attr,
                                bpfptr_t uattr)
{
        struct btf *btf;
        int err;

        if (!attr->func_info_cnt && !attr->line_info_cnt) {
                if (check_abnormal_return(env))
                        return -EINVAL;
                return 0;
        }

        btf = btf_get_by_fd(attr->prog_btf_fd);
        if (IS_ERR(btf))
                return PTR_ERR(btf);
        if (btf_is_kernel(btf)) {
                btf_put(btf);
                return -EACCES;
        }
        env->prog->aux->btf = btf;

        err = check_btf_func_early(env, attr, uattr);
        if (err)
                return err;
        return 0;
}

static int check_btf_info(struct bpf_verifier_env *env,
                          const union bpf_attr *attr,
                          bpfptr_t uattr)
{
        int err;

        if (!attr->func_info_cnt && !attr->line_info_cnt) {
                if (check_abnormal_return(env))
                        return -EINVAL;
                return 0;
        }

        err = check_btf_func(env, attr, uattr);
        if (err)
                return err;

        err = check_btf_line(env, attr, uattr);
        if (err)
                return err;

        err = check_core_relo(env, attr, uattr);
        if (err)
                return err;

        return 0;
}

/* check %cur's range satisfies %old's */
static bool range_within(const struct bpf_reg_state *old,
                         const struct bpf_reg_state *cur)
{
        return old->umin_value <= cur->umin_value &&
               old->umax_value >= cur->umax_value &&
               old->smin_value <= cur->smin_value &&
               old->smax_value >= cur->smax_value &&
               old->u32_min_value <= cur->u32_min_value &&
               old->u32_max_value >= cur->u32_max_value &&
               old->s32_min_value <= cur->s32_min_value &&
               old->s32_max_value >= cur->s32_max_value;
}

/* If in the old state two registers had the same id, then they need to have
 * the same id in the new state as well.  But that id could be different from
 * the old state, so we need to track the mapping from old to new ids.
 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
 * regs with old id 5 must also have new id 9 for the new state to be safe.  But
 * regs with a different old id could still have new id 9, we don't care about
 * that.
 * So we look through our idmap to see if this old id has been seen before.  If
 * so, we require the new id to match; otherwise, we add the id pair to the map.
 */
static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
{
        struct bpf_id_pair *map = idmap->map;
        unsigned int i;

        /* either both IDs should be set or both should be zero */
        if (!!old_id != !!cur_id)
                return false;

        if (old_id == 0) /* cur_id == 0 as well */
                return true;

        for (i = 0; i < BPF_ID_MAP_SIZE; i++) {
                if (!map[i].old) {
                        /* Reached an empty slot; haven't seen this id before */
                        map[i].old = old_id;
                        map[i].cur = cur_id;
                        return true;
                }
                if (map[i].old == old_id)
                        return map[i].cur == cur_id;
                if (map[i].cur == cur_id)
                        return false;
        }
        /* We ran out of idmap slots, which should be impossible */
        WARN_ON_ONCE(1);
        return false;
}

/* Similar to check_ids(), but allocate a unique temporary ID
 * for 'old_id' or 'cur_id' of zero.
 * This makes pairs like '0 vs unique ID', 'unique ID vs 0' valid.
 */
static bool check_scalar_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
{
        old_id = old_id ? old_id : ++idmap->tmp_id_gen;
        cur_id = cur_id ? cur_id : ++idmap->tmp_id_gen;

        return check_ids(old_id, cur_id, idmap);
}

static void clean_func_state(struct bpf_verifier_env *env,
                             struct bpf_func_state *st)
{
        enum bpf_reg_liveness live;
        int i, j;

        for (i = 0; i < BPF_REG_FP; i++) {
                live = st->regs[i].live;
                /* liveness must not touch this register anymore */
                st->regs[i].live |= REG_LIVE_DONE;
                if (!(live & REG_LIVE_READ))
                        /* since the register is unused, clear its state
                         * to make further comparison simpler
                         */
                        __mark_reg_not_init(env, &st->regs[i]);
        }

        for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
                live = st->stack[i].spilled_ptr.live;
                /* liveness must not touch this stack slot anymore */
                st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
                if (!(live & REG_LIVE_READ)) {
                        __mark_reg_not_init(env, &st->stack[i].spilled_ptr);
                        for (j = 0; j < BPF_REG_SIZE; j++)
                                st->stack[i].slot_type[j] = STACK_INVALID;
                }
        }
}

static void clean_verifier_state(struct bpf_verifier_env *env,
                                 struct bpf_verifier_state *st)
{
        int i;

        if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
                /* all regs in this state in all frames were already marked */
                return;

        for (i = 0; i <= st->curframe; i++)
                clean_func_state(env, st->frame[i]);
}

/* the parentage chains form a tree.
 * the verifier states are added to state lists at given insn and
 * pushed into state stack for future exploration.
 * when the verifier reaches bpf_exit insn some of the verifer states
 * stored in the state lists have their final liveness state already,
 * but a lot of states will get revised from liveness point of view when
 * the verifier explores other branches.
 * Example:
 * 1: r0 = 1
 * 2: if r1 == 100 goto pc+1
 * 3: r0 = 2
 * 4: exit
 * when the verifier reaches exit insn the register r0 in the state list of
 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch
 * of insn 2 and goes exploring further. At the insn 4 it will walk the
 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ.
 *
 * Since the verifier pushes the branch states as it sees them while exploring
 * the program the condition of walking the branch instruction for the second
 * time means that all states below this branch were already explored and
 * their final liveness marks are already propagated.
 * Hence when the verifier completes the search of state list in is_state_visited()
 * we can call this clean_live_states() function to mark all liveness states
 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state'
 * will not be used.
 * This function also clears the registers and stack for states that !READ
 * to simplify state merging.
 *
 * Important note here that walking the same branch instruction in the callee
 * doesn't meant that the states are DONE. The verifier has to compare
 * the callsites
 */
static void clean_live_states(struct bpf_verifier_env *env, int insn,
                              struct bpf_verifier_state *cur)
{
        struct bpf_verifier_state_list *sl;

        sl = *explored_state(env, insn);
        while (sl) {
                if (sl->state.branches)
                        goto next;
                if (sl->state.insn_idx != insn ||
                    !same_callsites(&sl->state, cur))
                        goto next;
                clean_verifier_state(env, &sl->state);
next:
                sl = sl->next;
        }
}

static bool regs_exact(const struct bpf_reg_state *rold,
                       const struct bpf_reg_state *rcur,
                       struct bpf_idmap *idmap)
{
        return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
               check_ids(rold->id, rcur->id, idmap) &&
               check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap);
}

enum exact_level {
        NOT_EXACT,
        EXACT,
        RANGE_WITHIN
};

/* Returns true if (rold safe implies rcur safe) */
static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
                    struct bpf_reg_state *rcur, struct bpf_idmap *idmap,
                    enum exact_level exact)
{
        if (exact == EXACT)
                return regs_exact(rold, rcur, idmap);

        if (!(rold->live & REG_LIVE_READ) && exact == NOT_EXACT)
                /* explored state didn't use this */
                return true;
        if (rold->type == NOT_INIT) {
                if (exact == NOT_EXACT || rcur->type == NOT_INIT)
                        /* explored state can't have used this */
                        return true;
        }

        /* Enforce that register types have to match exactly, including their
         * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general
         * rule.
         *
         * One can make a point that using a pointer register as unbounded
         * SCALAR would be technically acceptable, but this could lead to
         * pointer leaks because scalars are allowed to leak while pointers
         * are not. We could make this safe in special cases if root is
         * calling us, but it's probably not worth the hassle.
         *
         * Also, register types that are *not* MAYBE_NULL could technically be
         * safe to use as their MAYBE_NULL variants (e.g., PTR_TO_MAP_VALUE
         * is safe to be used as PTR_TO_MAP_VALUE_OR_NULL, provided both point
         * to the same map).
         * However, if the old MAYBE_NULL register then got NULL checked,
         * doing so could have affected others with the same id, and we can't
         * check for that because we lost the id when we converted to
         * a non-MAYBE_NULL variant.
         * So, as a general rule we don't allow mixing MAYBE_NULL and
         * non-MAYBE_NULL registers as well.
         */
        if (rold->type != rcur->type)
                return false;

        switch (base_type(rold->type)) {
        case SCALAR_VALUE:
                if (env->explore_alu_limits) {
                        /* explore_alu_limits disables tnum_in() and range_within()
                         * logic and requires everything to be strict
                         */
                        return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
                               check_scalar_ids(rold->id, rcur->id, idmap);
                }
                if (!rold->precise && exact == NOT_EXACT)
                        return true;
                if ((rold->id & BPF_ADD_CONST) != (rcur->id & BPF_ADD_CONST))
                        return false;
                if ((rold->id & BPF_ADD_CONST) && (rold->off != rcur->off))
                        return false;
                /* Why check_ids() for scalar registers?
                 *
                 * Consider the following BPF code:
                 *   1: r6 = ... unbound scalar, ID=a ...
                 *   2: r7 = ... unbound scalar, ID=b ...
                 *   3: if (r6 > r7) goto +1
                 *   4: r6 = r7
                 *   5: if (r6 > X) goto ...
                 *   6: ... memory operation using r7 ...
                 *
                 * First verification path is [1-6]:
                 * - at (4) same bpf_reg_state::id (b) would be assigned to r6 and r7;
                 * - at (5) r6 would be marked <= X, sync_linked_regs() would also mark
                 *   r7 <= X, because r6 and r7 share same id.
                 * Next verification path is [1-4, 6].
                 *
                 * Instruction (6) would be reached in two states:
                 *   I.  r6{.id=b}, r7{.id=b} via path 1-6;
                 *   II. r6{.id=a}, r7{.id=b} via path 1-4, 6.
                 *
                 * Use check_ids() to distinguish these states.
                 * ---
                 * Also verify that new value satisfies old value range knowledge.
                 */
                return range_within(rold, rcur) &&
                       tnum_in(rold->var_off, rcur->var_off) &&
                       check_scalar_ids(rold->id, rcur->id, idmap);
        case PTR_TO_MAP_KEY:
        case PTR_TO_MAP_VALUE:
        case PTR_TO_MEM:
        case PTR_TO_BUF:
        case PTR_TO_TP_BUFFER:
                /* If the new min/max/var_off satisfy the old ones and
                 * everything else matches, we are OK.
                 */
                return memcmp(rold, rcur, offsetof(struct bpf_reg_state, var_off)) == 0 &&
                       range_within(rold, rcur) &&
                       tnum_in(rold->var_off, rcur->var_off) &&
                       check_ids(rold->id, rcur->id, idmap) &&
                       check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap);
        case PTR_TO_PACKET_META:
        case PTR_TO_PACKET:
                /* We must have at least as much range as the old ptr
                 * did, so that any accesses which were safe before are
                 * still safe.  This is true even if old range < old off,
                 * since someone could have accessed through (ptr - k), or
                 * even done ptr -= k in a register, to get a safe access.
                 */
                if (rold->range > rcur->range)
                        return false;
                /* If the offsets don't match, we can't trust our alignment;
                 * nor can we be sure that we won't fall out of range.
                 */
                if (rold->off != rcur->off)
                        return false;
                /* id relations must be preserved */
                if (!check_ids(rold->id, rcur->id, idmap))
                        return false;
                /* new val must satisfy old val knowledge */
                return range_within(rold, rcur) &&
                       tnum_in(rold->var_off, rcur->var_off);
        case PTR_TO_STACK:
                /* two stack pointers are equal only if they're pointing to
                 * the same stack frame, since fp-8 in foo != fp-8 in bar
                 */
                return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno;
        case PTR_TO_ARENA:
                return true;
        default:
                return regs_exact(rold, rcur, idmap);
        }
}

static struct bpf_reg_state unbound_reg;

static __init int unbound_reg_init(void)
{
        __mark_reg_unknown_imprecise(&unbound_reg);
        unbound_reg.live |= REG_LIVE_READ;
        return 0;
}
late_initcall(unbound_reg_init);

static bool is_stack_all_misc(struct bpf_verifier_env *env,
                              struct bpf_stack_state *stack)
{
        u32 i;

        for (i = 0; i < ARRAY_SIZE(stack->slot_type); ++i) {
                if ((stack->slot_type[i] == STACK_MISC) ||
                    (stack->slot_type[i] == STACK_INVALID && env->allow_uninit_stack))
                        continue;
                return false;
        }

        return true;
}

static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env,
                                                  struct bpf_stack_state *stack)
{
        if (is_spilled_scalar_reg64(stack))
                return &stack->spilled_ptr;

        if (is_stack_all_misc(env, stack))
                return &unbound_reg;

        return NULL;
}

static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
                      struct bpf_func_state *cur, struct bpf_idmap *idmap,
                      enum exact_level exact)
{
        int i, spi;

        /* walk slots of the explored stack and ignore any additional
         * slots in the current stack, since explored(safe) state
         * didn't use them
         */
        for (i = 0; i < old->allocated_stack; i++) {
                struct bpf_reg_state *old_reg, *cur_reg;

                spi = i / BPF_REG_SIZE;

                if (exact != NOT_EXACT &&
                    (i >= cur->allocated_stack ||
                     old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
                     cur->stack[spi].slot_type[i % BPF_REG_SIZE]))
                        return false;

                if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)
                    && exact == NOT_EXACT) {
                        i += BPF_REG_SIZE - 1;
                        /* explored state didn't use this */
                        continue;
                }

                if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
                        continue;

                if (env->allow_uninit_stack &&
                    old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC)
                        continue;

                /* explored stack has more populated slots than current stack
                 * and these slots were used
                 */
                if (i >= cur->allocated_stack)
                        return false;

                /* 64-bit scalar spill vs all slots MISC and vice versa.
                 * Load from all slots MISC produces unbound scalar.
                 * Construct a fake register for such stack and call
                 * regsafe() to ensure scalar ids are compared.
                 */
                old_reg = scalar_reg_for_stack(env, &old->stack[spi]);
                cur_reg = scalar_reg_for_stack(env, &cur->stack[spi]);
                if (old_reg && cur_reg) {
                        if (!regsafe(env, old_reg, cur_reg, idmap, exact))
                                return false;
                        i += BPF_REG_SIZE - 1;
                        continue;
                }

                /* if old state was safe with misc data in the stack
                 * it will be safe with zero-initialized stack.
                 * The opposite is not true
                 */
                if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
                    cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
                        continue;
                if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
                    cur->stack[spi].slot_type[i % BPF_REG_SIZE])
                        /* Ex: old explored (safe) state has STACK_SPILL in
                         * this stack slot, but current has STACK_MISC ->
                         * this verifier states are not equivalent,
                         * return false to continue verification of this path
                         */
                        return false;
                if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1)
                        continue;
                /* Both old and cur are having same slot_type */
                switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) {
                case STACK_SPILL:
                        /* when explored and current stack slot are both storing
                         * spilled registers, check that stored pointers types
                         * are the same as well.
                         * Ex: explored safe path could have stored
                         * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8}
                         * but current path has stored:
                         * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
                         * such verifier states are not equivalent.
                         * return false to continue verification of this path
                         */
                        if (!regsafe(env, &old->stack[spi].spilled_ptr,
                                     &cur->stack[spi].spilled_ptr, idmap, exact))
                                return false;
                        break;
                case STACK_DYNPTR:
                        old_reg = &old->stack[spi].spilled_ptr;
                        cur_reg = &cur->stack[spi].spilled_ptr;
                        if (old_reg->dynptr.type != cur_reg->dynptr.type ||
                            old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot ||
                            !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap))
                                return false;
                        break;
                case STACK_ITER:
                        old_reg = &old->stack[spi].spilled_ptr;
                        cur_reg = &cur->stack[spi].spilled_ptr;
                        /* iter.depth is not compared between states as it
                         * doesn't matter for correctness and would otherwise
                         * prevent convergence; we maintain it only to prevent
                         * infinite loop check triggering, see
                         * iter_active_depths_differ()
                         */
                        if (old_reg->iter.btf != cur_reg->iter.btf ||
                            old_reg->iter.btf_id != cur_reg->iter.btf_id ||
                            old_reg->iter.state != cur_reg->iter.state ||
                            /* ignore {old_reg,cur_reg}->iter.depth, see above */
                            !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap))
                                return false;
                        break;
                case STACK_IRQ_FLAG:
                        old_reg = &old->stack[spi].spilled_ptr;
                        cur_reg = &cur->stack[spi].spilled_ptr;
                        if (!check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap))
                                return false;
                        break;
                case STACK_MISC:
                case STACK_ZERO:
                case STACK_INVALID:
                        continue;
                /* Ensure that new unhandled slot types return false by default */
                default:
                        return false;
                }
        }
        return true;
}

static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cur,
                    struct bpf_idmap *idmap)
{
        int i;

        if (old->acquired_refs != cur->acquired_refs)
                return false;

        if (old->active_locks != cur->active_locks)
                return false;

        if (old->active_preempt_locks != cur->active_preempt_locks)
                return false;

        if (old->active_rcu_lock != cur->active_rcu_lock)
                return false;

        if (!check_ids(old->active_irq_id, cur->active_irq_id, idmap))
                return false;

        for (i = 0; i < old->acquired_refs; i++) {
                if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap) ||
                    old->refs[i].type != cur->refs[i].type)
                        return false;
                switch (old->refs[i].type) {
                case REF_TYPE_PTR:
                case REF_TYPE_IRQ:
                        break;
                case REF_TYPE_LOCK:
                        if (old->refs[i].ptr != cur->refs[i].ptr)
                                return false;
                        break;
                default:
                        WARN_ONCE(1, "Unhandled enum type for reference state: %d\n", old->refs[i].type);
                        return false;
                }
        }

        return true;
}

/* compare two verifier states
 *
 * all states stored in state_list are known to be valid, since
 * verifier reached 'bpf_exit' instruction through them
 *
 * this function is called when verifier exploring different branches of
 * execution popped from the state stack. If it sees an old state that has
 * more strict register state and more strict stack state then this execution
 * branch doesn't need to be explored further, since verifier already
 * concluded that more strict state leads to valid finish.
 *
 * Therefore two states are equivalent if register state is more conservative
 * and explored stack state is more conservative than the current one.
 * Example:
 *       explored                   current
 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC)
 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC)
 *
 * In other words if current stack state (one being explored) has more
 * valid slots than old one that already passed validation, it means
 * the verifier can stop exploring and conclude that current state is valid too
 *
 * Similarly with registers. If explored state has register type as invalid
 * whereas register type in current state is meaningful, it means that
 * the current state will reach 'bpf_exit' instruction safely
 */
static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,
                              struct bpf_func_state *cur, enum exact_level exact)
{
        int i;

        if (old->callback_depth > cur->callback_depth)
                return false;

        for (i = 0; i < MAX_BPF_REG; i++)
                if (!regsafe(env, &old->regs[i], &cur->regs[i],
                             &env->idmap_scratch, exact))
                        return false;

        if (!stacksafe(env, old, cur, &env->idmap_scratch, exact))
                return false;

        return true;
}

static void reset_idmap_scratch(struct bpf_verifier_env *env)
{
        env->idmap_scratch.tmp_id_gen = env->id_gen;
        memset(&env->idmap_scratch.map, 0, sizeof(env->idmap_scratch.map));
}

static bool states_equal(struct bpf_verifier_env *env,
                         struct bpf_verifier_state *old,
                         struct bpf_verifier_state *cur,
                         enum exact_level exact)
{
        int i;

        if (old->curframe != cur->curframe)
                return false;

        reset_idmap_scratch(env);

        /* Verification state from speculative execution simulation
         * must never prune a non-speculative execution one.
         */
        if (old->speculative && !cur->speculative)
                return false;

        if (old->in_sleepable != cur->in_sleepable)
                return false;

        if (!refsafe(old, cur, &env->idmap_scratch))
                return false;

        /* for states to be equal callsites have to be the same
         * and all frame states need to be equivalent
         */
        for (i = 0; i <= old->curframe; i++) {
                if (old->frame[i]->callsite != cur->frame[i]->callsite)
                        return false;
                if (!func_states_equal(env, old->frame[i], cur->frame[i], exact))
                        return false;
        }
        return true;
}

/* Return 0 if no propagation happened. Return negative error code if error
 * happened. Otherwise, return the propagated bit.
 */
static int propagate_liveness_reg(struct bpf_verifier_env *env,
                                  struct bpf_reg_state *reg,
                                  struct bpf_reg_state *parent_reg)
{
        u8 parent_flag = parent_reg->live & REG_LIVE_READ;
        u8 flag = reg->live & REG_LIVE_READ;
        int err;

        /* When comes here, read flags of PARENT_REG or REG could be any of
         * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need
         * of propagation if PARENT_REG has strongest REG_LIVE_READ64.
         */
        if (parent_flag == REG_LIVE_READ64 ||
            /* Or if there is no read flag from REG. */
            !flag ||
            /* Or if the read flag from REG is the same as PARENT_REG. */
            parent_flag == flag)
                return 0;

        err = mark_reg_read(env, reg, parent_reg, flag);
        if (err)
                return err;

        return flag;
}

/* A write screens off any subsequent reads; but write marks come from the
 * straight-line code between a state and its parent.  When we arrive at an
 * equivalent state (jump target or such) we didn't arrive by the straight-line
 * code, so read marks in the state must propagate to the parent regardless
 * of the state's write marks. That's what 'parent == state->parent' comparison
 * in mark_reg_read() is for.
 */
static int propagate_liveness(struct bpf_verifier_env *env,
                              const struct bpf_verifier_state *vstate,
                              struct bpf_verifier_state *vparent)
{
        struct bpf_reg_state *state_reg, *parent_reg;
        struct bpf_func_state *state, *parent;
        int i, frame, err = 0;

        if (vparent->curframe != vstate->curframe) {
                WARN(1, "propagate_live: parent frame %d current frame %d\n",
                     vparent->curframe, vstate->curframe);
                return -EFAULT;
        }
        /* Propagate read liveness of registers... */
        BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
        for (frame = 0; frame <= vstate->curframe; frame++) {
                parent = vparent->frame[frame];
                state = vstate->frame[frame];
                parent_reg = parent->regs;
                state_reg = state->regs;
                /* We don't need to worry about FP liveness, it's read-only */
                for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) {
                        err = propagate_liveness_reg(env, &state_reg[i],
                                                     &parent_reg[i]);
                        if (err < 0)
                                return err;
                        if (err == REG_LIVE_READ64)
                                mark_insn_zext(env, &parent_reg[i]);
                }

                /* Propagate stack slots. */
                for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
                            i < parent->allocated_stack / BPF_REG_SIZE; i++) {
                        parent_reg = &parent->stack[i].spilled_ptr;
                        state_reg = &state->stack[i].spilled_ptr;
                        err = propagate_liveness_reg(env, state_reg,
                                                     parent_reg);
                        if (err < 0)
                                return err;
                }
        }
        return 0;
}

/* find precise scalars in the previous equivalent state and
 * propagate them into the current state
 */
static int propagate_precision(struct bpf_verifier_env *env,
                               const struct bpf_verifier_state *old)
{
        struct bpf_reg_state *state_reg;
        struct bpf_func_state *state;
        int i, err = 0, fr;
        bool first;

        for (fr = old->curframe; fr >= 0; fr--) {
                state = old->frame[fr];
                state_reg = state->regs;
                first = true;
                for (i = 0; i < BPF_REG_FP; i++, state_reg++) {
                        if (state_reg->type != SCALAR_VALUE ||
                            !state_reg->precise ||
                            !(state_reg->live & REG_LIVE_READ))
                                continue;
                        if (env->log.level & BPF_LOG_LEVEL2) {
                                if (first)
                                        verbose(env, "frame %d: propagating r%d", fr, i);
                                else
                                        verbose(env, ",r%d", i);
                        }
                        bt_set_frame_reg(&env->bt, fr, i);
                        first = false;
                }

                for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
                        if (!is_spilled_reg(&state->stack[i]))
                                continue;
                        state_reg = &state->stack[i].spilled_ptr;
                        if (state_reg->type != SCALAR_VALUE ||
                            !state_reg->precise ||
                            !(state_reg->live & REG_LIVE_READ))
                                continue;
                        if (env->log.level & BPF_LOG_LEVEL2) {
                                if (first)
                                        verbose(env, "frame %d: propagating fp%d",
                                                fr, (-i - 1) * BPF_REG_SIZE);
                                else
                                        verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE);
                        }
                        bt_set_frame_slot(&env->bt, fr, i);
                        first = false;
                }
                if (!first)
                        verbose(env, "\n");
        }

        err = mark_chain_precision_batch(env);
        if (err < 0)
                return err;

        return 0;
}

static bool states_maybe_looping(struct bpf_verifier_state *old,
                                 struct bpf_verifier_state *cur)
{
        struct bpf_func_state *fold, *fcur;
        int i, fr = cur->curframe;

        if (old->curframe != fr)
                return false;

        fold = old->frame[fr];
        fcur = cur->frame[fr];
        for (i = 0; i < MAX_BPF_REG; i++)
                if (memcmp(&fold->regs[i], &fcur->regs[i],
                           offsetof(struct bpf_reg_state, parent)))
                        return false;
        return true;
}

static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx)
{
        return env->insn_aux_data[insn_idx].is_iter_next;
}

/* is_state_visited() handles iter_next() (see process_iter_next_call() for
 * terminology) calls specially: as opposed to bounded BPF loops, it *expects*
 * states to match, which otherwise would look like an infinite loop. So while
 * iter_next() calls are taken care of, we still need to be careful and
 * prevent erroneous and too eager declaration of "ininite loop", when
 * iterators are involved.
 *
 * Here's a situation in pseudo-BPF assembly form:
 *
 *   0: again:                          ; set up iter_next() call args
 *   1:   r1 = &it                      ; <CHECKPOINT HERE>
 *   2:   call bpf_iter_num_next        ; this is iter_next() call
 *   3:   if r0 == 0 goto done
 *   4:   ... something useful here ...
 *   5:   goto again                    ; another iteration
 *   6: done:
 *   7:   r1 = &it
 *   8:   call bpf_iter_num_destroy     ; clean up iter state
 *   9:   exit
 *
 * This is a typical loop. Let's assume that we have a prune point at 1:,
 * before we get to `call bpf_iter_num_next` (e.g., because of that `goto
 * again`, assuming other heuristics don't get in a way).
 *
 * When we first time come to 1:, let's say we have some state X. We proceed
 * to 2:, fork states, enqueue ACTIVE, validate NULL case successfully, exit.
 * Now we come back to validate that forked ACTIVE state. We proceed through
 * 3-5, come to goto, jump to 1:. Let's assume our state didn't change, so we
 * are converging. But the problem is that we don't know that yet, as this
 * convergence has to happen at iter_next() call site only. So if nothing is
 * done, at 1: verifier will use bounded loop logic and declare infinite
 * looping (and would be *technically* correct, if not for iterator's
 * "eventual sticky NULL" contract, see process_iter_next_call()). But we
 * don't want that. So what we do in process_iter_next_call() when we go on
 * another ACTIVE iteration, we bump slot->iter.depth, to mark that it's
 * a different iteration. So when we suspect an infinite loop, we additionally
 * check if any of the *ACTIVE* iterator states depths differ. If yes, we
 * pretend we are not looping and wait for next iter_next() call.
 *
 * This only applies to ACTIVE state. In DRAINED state we don't expect to
 * loop, because that would actually mean infinite loop, as DRAINED state is
 * "sticky", and so we'll keep returning into the same instruction with the
 * same state (at least in one of possible code paths).
 *
 * This approach allows to keep infinite loop heuristic even in the face of
 * active iterator. E.g., C snippet below is and will be detected as
 * inifintely looping:
 *
 *   struct bpf_iter_num it;
 *   int *p, x;
 *
 *   bpf_iter_num_new(&it, 0, 10);
 *   while ((p = bpf_iter_num_next(&t))) {
 *       x = p;
 *       while (x--) {} // <<-- infinite loop here
 *   }
 *
 */
static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf_verifier_state *cur)
{
        struct bpf_reg_state *slot, *cur_slot;
        struct bpf_func_state *state;
        int i, fr;

        for (fr = old->curframe; fr >= 0; fr--) {
                state = old->frame[fr];
                for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
                        if (state->stack[i].slot_type[0] != STACK_ITER)
                                continue;

                        slot = &state->stack[i].spilled_ptr;
                        if (slot->iter.state != BPF_ITER_STATE_ACTIVE)
                                continue;

                        cur_slot = &cur->frame[fr]->stack[i].spilled_ptr;
                        if (cur_slot->iter.depth != slot->iter.depth)
                                return true;
                }
        }
        return false;
}

static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
{
        struct bpf_verifier_state_list *new_sl;
        struct bpf_verifier_state_list *sl, **pprev;
        struct bpf_verifier_state *cur = env->cur_state, *new, *loop_entry;
        int i, j, n, err, states_cnt = 0;
        bool force_new_state, add_new_state, force_exact;

        force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) ||
                          /* Avoid accumulating infinitely long jmp history */
                          cur->insn_hist_end - cur->insn_hist_start > 40;

        /* bpf progs typically have pruning point every 4 instructions
         * http://vger.kernel.org/bpfconf2019.html#session-1
         * Do not add new state for future pruning if the verifier hasn't seen
         * at least 2 jumps and at least 8 instructions.
         * This heuristics helps decrease 'total_states' and 'peak_states' metric.
         * In tests that amounts to up to 50% reduction into total verifier
         * memory consumption and 20% verifier time speedup.
         */
        add_new_state = force_new_state;
        if (env->jmps_processed - env->prev_jmps_processed >= 2 &&
            env->insn_processed - env->prev_insn_processed >= 8)
                add_new_state = true;

        pprev = explored_state(env, insn_idx);
        sl = *pprev;

        clean_live_states(env, insn_idx, cur);

        while (sl) {
                states_cnt++;
                if (sl->state.insn_idx != insn_idx)
                        goto next;

                if (sl->state.branches) {
                        struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];

                        if (frame->in_async_callback_fn &&
                            frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) {
                                /* Different async_entry_cnt means that the verifier is
                                 * processing another entry into async callback.
                                 * Seeing the same state is not an indication of infinite
                                 * loop or infinite recursion.
                                 * But finding the same state doesn't mean that it's safe
                                 * to stop processing the current state. The previous state
                                 * hasn't yet reached bpf_exit, since state.branches > 0.
                                 * Checking in_async_callback_fn alone is not enough either.
                                 * Since the verifier still needs to catch infinite loops
                                 * inside async callbacks.
                                 */
                                goto skip_inf_loop_check;
                        }
                        /* BPF open-coded iterators loop detection is special.
                         * states_maybe_looping() logic is too simplistic in detecting
                         * states that *might* be equivalent, because it doesn't know
                         * about ID remapping, so don't even perform it.
                         * See process_iter_next_call() and iter_active_depths_differ()
                         * for overview of the logic. When current and one of parent
                         * states are detected as equivalent, it's a good thing: we prove
                         * convergence and can stop simulating further iterations.
                         * It's safe to assume that iterator loop will finish, taking into
                         * account iter_next() contract of eventually returning
                         * sticky NULL result.
                         *
                         * Note, that states have to be compared exactly in this case because
                         * read and precision marks might not be finalized inside the loop.
                         * E.g. as in the program below:
                         *
                         *     1. r7 = -16
                         *     2. r6 = bpf_get_prandom_u32()
                         *     3. while (bpf_iter_num_next(&fp[-8])) {
                         *     4.   if (r6 != 42) {
                         *     5.     r7 = -32
                         *     6.     r6 = bpf_get_prandom_u32()
                         *     7.     continue
                         *     8.   }
                         *     9.   r0 = r10
                         *    10.   r0 += r7
                         *    11.   r8 = *(u64 *)(r0 + 0)
                         *    12.   r6 = bpf_get_prandom_u32()
                         *    13. }
                         *
                         * Here verifier would first visit path 1-3, create a checkpoint at 3
                         * with r7=-16, continue to 4-7,3. Existing checkpoint at 3 does
                         * not have read or precision mark for r7 yet, thus inexact states
                         * comparison would discard current state with r7=-32
                         * => unsafe memory access at 11 would not be caught.
                         */
                        if (is_iter_next_insn(env, insn_idx)) {
                                if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
                                        struct bpf_func_state *cur_frame;
                                        struct bpf_reg_state *iter_state, *iter_reg;
                                        int spi;

                                        cur_frame = cur->frame[cur->curframe];
                                        /* btf_check_iter_kfuncs() enforces that
                                         * iter state pointer is always the first arg
                                         */
                                        iter_reg = &cur_frame->regs[BPF_REG_1];
                                        /* current state is valid due to states_equal(),
                                         * so we can assume valid iter and reg state,
                                         * no need for extra (re-)validations
                                         */
                                        spi = __get_spi(iter_reg->off + iter_reg->var_off.value);
                                        iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;
                                        if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) {
                                                update_loop_entry(cur, &sl->state);
                                                goto hit;
                                        }
                                }
                                goto skip_inf_loop_check;
                        }
                        if (is_may_goto_insn_at(env, insn_idx)) {
                                if (sl->state.may_goto_depth != cur->may_goto_depth &&
                                    states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
                                        update_loop_entry(cur, &sl->state);
                                        goto hit;
                                }
                        }
                        if (calls_callback(env, insn_idx)) {
                                if (states_equal(env, &sl->state, cur, RANGE_WITHIN))
                                        goto hit;
                                goto skip_inf_loop_check;
                        }
                        /* attempt to detect infinite loop to avoid unnecessary doomed work */
                        if (states_maybe_looping(&sl->state, cur) &&
                            states_equal(env, &sl->state, cur, EXACT) &&
                            !iter_active_depths_differ(&sl->state, cur) &&
                            sl->state.may_goto_depth == cur->may_goto_depth &&
                            sl->state.callback_unroll_depth == cur->callback_unroll_depth) {
                                verbose_linfo(env, insn_idx, "; ");
                                verbose(env, "infinite loop detected at insn %d\n", insn_idx);
                                verbose(env, "cur state:");
                                print_verifier_state(env, cur, cur->curframe, true);
                                verbose(env, "old state:");
                                print_verifier_state(env, &sl->state, cur->curframe, true);
                                return -EINVAL;
                        }
                        /* if the verifier is processing a loop, avoid adding new state
                         * too often, since different loop iterations have distinct
                         * states and may not help future pruning.
                         * This threshold shouldn't be too low to make sure that
                         * a loop with large bound will be rejected quickly.
                         * The most abusive loop will be:
                         * r1 += 1
                         * if r1 < 1000000 goto pc-2
                         * 1M insn_procssed limit / 100 == 10k peak states.
                         * This threshold shouldn't be too high either, since states
                         * at the end of the loop are likely to be useful in pruning.
                         */
skip_inf_loop_check:
                        if (!force_new_state &&
                            env->jmps_processed - env->prev_jmps_processed < 20 &&
                            env->insn_processed - env->prev_insn_processed < 100)
                                add_new_state = false;
                        goto miss;
                }
                /* If sl->state is a part of a loop and this loop's entry is a part of
                 * current verification path then states have to be compared exactly.
                 * 'force_exact' is needed to catch the following case:
                 *
                 *                initial     Here state 'succ' was processed first,
                 *                  |         it was eventually tracked to produce a
                 *                  V         state identical to 'hdr'.
                 *     .---------> hdr        All branches from 'succ' had been explored
                 *     |            |         and thus 'succ' has its .branches == 0.
                 *     |            V
                 *     |    .------...        Suppose states 'cur' and 'succ' correspond
                 *     |    |       |         to the same instruction + callsites.
                 *     |    V       V         In such case it is necessary to check
                 *     |   ...     ...        if 'succ' and 'cur' are states_equal().
                 *     |    |       |         If 'succ' and 'cur' are a part of the
                 *     |    V       V         same loop exact flag has to be set.
                 *     |   succ <- cur        To check if that is the case, verify
                 *     |    |                 if loop entry of 'succ' is in current
                 *     |    V                 DFS path.
                 *     |   ...
                 *     |    |
                 *     '----'
                 *
                 * Additional details are in the comment before get_loop_entry().
                 */
                loop_entry = get_loop_entry(&sl->state);
                force_exact = loop_entry && loop_entry->branches > 0;
                if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) {
                        if (force_exact)
                                update_loop_entry(cur, loop_entry);
hit:
                        sl->hit_cnt++;
                        /* reached equivalent register/stack state,
                         * prune the search.
                         * Registers read by the continuation are read by us.
                         * If we have any write marks in env->cur_state, they
                         * will prevent corresponding reads in the continuation
                         * from reaching our parent (an explored_state).  Our
                         * own state will get the read marks recorded, but
                         * they'll be immediately forgotten as we're pruning
                         * this state and will pop a new one.
                         */
                        err = propagate_liveness(env, &sl->state, cur);

                        /* if previous state reached the exit with precision and
                         * current state is equivalent to it (except precision marks)
                         * the precision needs to be propagated back in
                         * the current state.
                         */
                        if (is_jmp_point(env, env->insn_idx))
                                err = err ? : push_insn_history(env, cur, 0, 0);
                        err = err ? : propagate_precision(env, &sl->state);
                        if (err)
                                return err;
                        return 1;
                }
miss:
                /* when new state is not going to be added do not increase miss count.
                 * Otherwise several loop iterations will remove the state
                 * recorded earlier. The goal of these heuristics is to have
                 * states from some iterations of the loop (some in the beginning
                 * and some at the end) to help pruning.
                 */
                if (add_new_state)
                        sl->miss_cnt++;
                /* heuristic to determine whether this state is beneficial
                 * to keep checking from state equivalence point of view.
                 * Higher numbers increase max_states_per_insn and verification time,
                 * but do not meaningfully decrease insn_processed.
                 * 'n' controls how many times state could miss before eviction.
                 * Use bigger 'n' for checkpoints because evicting checkpoint states
                 * too early would hinder iterator convergence.
                 */
                n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3;
                if (sl->miss_cnt > sl->hit_cnt * n + n) {
                        /* the state is unlikely to be useful. Remove it to
                         * speed up verification
                         */
                        *pprev = sl->next;
                        if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE &&
                            !sl->state.used_as_loop_entry) {
                                u32 br = sl->state.branches;

                                WARN_ONCE(br,
                                          "BUG live_done but branches_to_explore %d\n",
                                          br);
                                free_verifier_state(&sl->state, false);
                                kfree(sl);
                                env->peak_states--;
                        } else {
                                /* cannot free this state, since parentage chain may
                                 * walk it later. Add it for free_list instead to
                                 * be freed at the end of verification
                                 */
                                sl->next = env->free_list;
                                env->free_list = sl;
                        }
                        sl = *pprev;
                        continue;
                }
next:
                pprev = &sl->next;
                sl = *pprev;
        }

        if (env->max_states_per_insn < states_cnt)
                env->max_states_per_insn = states_cnt;

        if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
                return 0;

        if (!add_new_state)
                return 0;

        /* There were no equivalent states, remember the current one.
         * Technically the current state is not proven to be safe yet,
         * but it will either reach outer most bpf_exit (which means it's safe)
         * or it will be rejected. When there are no loops the verifier won't be
         * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
         * again on the way to bpf_exit.
         * When looping the sl->state.branches will be > 0 and this state
         * will not be considered for equivalence until branches == 0.
         */
        new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
        if (!new_sl)
                return -ENOMEM;
        env->total_states++;
        env->peak_states++;
        env->prev_jmps_processed = env->jmps_processed;
        env->prev_insn_processed = env->insn_processed;

        /* forget precise markings we inherited, see __mark_chain_precision */
        if (env->bpf_capable)
                mark_all_scalars_imprecise(env, cur);

        /* add new state to the head of linked list */
        new = &new_sl->state;
        err = copy_verifier_state(new, cur);
        if (err) {
                free_verifier_state(new, false);
                kfree(new_sl);
                return err;
        }
        new->insn_idx = insn_idx;
        WARN_ONCE(new->branches != 1,
                  "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);

        cur->parent = new;
        cur->first_insn_idx = insn_idx;
        cur->insn_hist_start = cur->insn_hist_end;
        cur->dfs_depth = new->dfs_depth + 1;
        new_sl->next = *explored_state(env, insn_idx);
        *explored_state(env, insn_idx) = new_sl;
        /* connect new state to parentage chain. Current frame needs all
         * registers connected. Only r6 - r9 of the callers are alive (pushed
         * to the stack implicitly by JITs) so in callers' frames connect just
         * r6 - r9 as an optimization. Callers will have r1 - r5 connected to
         * the state of the call instruction (with WRITTEN set), and r0 comes
         * from callee with its full parentage chain, anyway.
         */
        /* clear write marks in current state: the writes we did are not writes
         * our child did, so they don't screen off its reads from us.
         * (There are no read marks in current state, because reads always mark
         * their parent and current state never has children yet.  Only
         * explored_states can get read marks.)
         */
        for (j = 0; j <= cur->curframe; j++) {
                for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
                        cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
                for (i = 0; i < BPF_REG_FP; i++)
                        cur->frame[j]->regs[i].live = REG_LIVE_NONE;
        }

        /* all stack frames are accessible from callee, clear them all */
        for (j = 0; j <= cur->curframe; j++) {
                struct bpf_func_state *frame = cur->frame[j];
                struct bpf_func_state *newframe = new->frame[j];

                for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
                        frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
                        frame->stack[i].spilled_ptr.parent =
                                                &newframe->stack[i].spilled_ptr;
                }
        }
        return 0;
}

/* Return true if it's OK to have the same insn return a different type. */
static bool reg_type_mismatch_ok(enum bpf_reg_type type)
{
        switch (base_type(type)) {
        case PTR_TO_CTX:
        case PTR_TO_SOCKET:
        case PTR_TO_SOCK_COMMON:
        case PTR_TO_TCP_SOCK:
        case PTR_TO_XDP_SOCK:
        case PTR_TO_BTF_ID:
        case PTR_TO_ARENA:
                return false;
        default:
                return true;
        }
}

/* If an instruction was previously used with particular pointer types, then we
 * need to be careful to avoid cases such as the below, where it may be ok
 * for one branch accessing the pointer, but not ok for the other branch:
 *
 * R1 = sock_ptr
 * goto X;
 * ...
 * R1 = some_other_valid_ptr;
 * goto X;
 * ...
 * R2 = *(u32 *)(R1 + 0);
 */
static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
{
        return src != prev && (!reg_type_mismatch_ok(src) ||
                               !reg_type_mismatch_ok(prev));
}

static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
                             bool allow_trust_mismatch)
{
        enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type;

        if (*prev_type == NOT_INIT) {
                /* Saw a valid insn
                 * dst_reg = *(u32 *)(src_reg + off)
                 * save type to validate intersecting paths
                 */
                *prev_type = type;
        } else if (reg_type_mismatch(type, *prev_type)) {
                /* Abuser program is trying to use the same insn
                 * dst_reg = *(u32*) (src_reg + off)
                 * with different pointer types:
                 * src_reg == ctx in one branch and
                 * src_reg == stack|map in some other branch.
                 * Reject it.
                 */
                if (allow_trust_mismatch &&
                    base_type(type) == PTR_TO_BTF_ID &&
                    base_type(*prev_type) == PTR_TO_BTF_ID) {
                        /*
                         * Have to support a use case when one path through
                         * the program yields TRUSTED pointer while another
                         * is UNTRUSTED. Fallback to UNTRUSTED to generate
                         * BPF_PROBE_MEM/BPF_PROBE_MEMSX.
                         */
                        *prev_type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
                } else {
                        verbose(env, "same insn cannot be used with different pointers\n");
                        return -EINVAL;
                }
        }

        return 0;
}

static int do_check(struct bpf_verifier_env *env)
{
        bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
        struct bpf_verifier_state *state = env->cur_state;
        struct bpf_insn *insns = env->prog->insnsi;
        struct bpf_reg_state *regs;
        int insn_cnt = env->prog->len;
        bool do_print_state = false;
        int prev_insn_idx = -1;

        for (;;) {
                bool exception_exit = false;
                struct bpf_insn *insn;
                u8 class;
                int err;

                /* reset current history entry on each new instruction */
                env->cur_hist_ent = NULL;

                env->prev_insn_idx = prev_insn_idx;
                if (env->insn_idx >= insn_cnt) {
                        verbose(env, "invalid insn idx %d insn_cnt %d\n",
                                env->insn_idx, insn_cnt);
                        return -EFAULT;
                }

                insn = &insns[env->insn_idx];
                class = BPF_CLASS(insn->code);

                if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
                        verbose(env,
                                "BPF program is too large. Processed %d insn\n",
                                env->insn_processed);
                        return -E2BIG;
                }

                state->last_insn_idx = env->prev_insn_idx;

                if (is_prune_point(env, env->insn_idx)) {
                        err = is_state_visited(env, env->insn_idx);
                        if (err < 0)
                                return err;
                        if (err == 1) {
                                /* found equivalent state, can prune the search */
                                if (env->log.level & BPF_LOG_LEVEL) {
                                        if (do_print_state)
                                                verbose(env, "\nfrom %d to %d%s: safe\n",
                                                        env->prev_insn_idx, env->insn_idx,
                                                        env->cur_state->speculative ?
                                                        " (speculative execution)" : "");
                                        else
                                                verbose(env, "%d: safe\n", env->insn_idx);
                                }
                                goto process_bpf_exit;
                        }
                }

                if (is_jmp_point(env, env->insn_idx)) {
                        err = push_insn_history(env, state, 0, 0);
                        if (err)
                                return err;
                }

                if (signal_pending(current))
                        return -EAGAIN;

                if (need_resched())
                        cond_resched();

                if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {
                        verbose(env, "\nfrom %d to %d%s:",
                                env->prev_insn_idx, env->insn_idx,
                                env->cur_state->speculative ?
                                " (speculative execution)" : "");
                        print_verifier_state(env, state, state->curframe, true);
                        do_print_state = false;
                }

                if (env->log.level & BPF_LOG_LEVEL) {
                        const struct bpf_insn_cbs cbs = {
                                .cb_call        = disasm_kfunc_name,
                                .cb_print        = verbose,
                                .private_data        = env,
                        };

                        if (verifier_state_scratched(env))
                                print_insn_state(env, state, state->curframe);

                        verbose_linfo(env, env->insn_idx, "; ");
                        env->prev_log_pos = env->log.end_pos;
                        verbose(env, "%d: ", env->insn_idx);
                        print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
                        env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;
                        env->prev_log_pos = env->log.end_pos;
                }

                if (bpf_prog_is_offloaded(env->prog->aux)) {
                        err = bpf_prog_offload_verify_insn(env, env->insn_idx,
                                                           env->prev_insn_idx);
                        if (err)
                                return err;
                }

                regs = cur_regs(env);
                sanitize_mark_insn_seen(env);
                prev_insn_idx = env->insn_idx;

                if (class == BPF_ALU || class == BPF_ALU64) {
                        err = check_alu_op(env, insn);
                        if (err)
                                return err;

                } else if (class == BPF_LDX) {
                        enum bpf_reg_type src_reg_type;

                        /* check for reserved fields is already done */

                        /* check src operand */
                        err = check_reg_arg(env, insn->src_reg, SRC_OP);
                        if (err)
                                return err;

                        err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
                        if (err)
                                return err;

                        src_reg_type = regs[insn->src_reg].type;

                        /* check that memory (src_reg + off) is readable,
                         * the state of dst_reg will be updated by this func
                         */
                        err = check_mem_access(env, env->insn_idx, insn->src_reg,
                                               insn->off, BPF_SIZE(insn->code),
                                               BPF_READ, insn->dst_reg, false,
                                               BPF_MODE(insn->code) == BPF_MEMSX);
                        err = err ?: save_aux_ptr_type(env, src_reg_type, true);
                        err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], "ldx");
                        if (err)
                                return err;
                } else if (class == BPF_STX) {
                        enum bpf_reg_type dst_reg_type;

                        if (BPF_MODE(insn->code) == BPF_ATOMIC) {
                                err = check_atomic(env, env->insn_idx, insn);
                                if (err)
                                        return err;
                                env->insn_idx++;
                                continue;
                        }

                        if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) {
                                verbose(env, "BPF_STX uses reserved fields\n");
                                return -EINVAL;
                        }

                        /* check src1 operand */
                        err = check_reg_arg(env, insn->src_reg, SRC_OP);
                        if (err)
                                return err;
                        /* check src2 operand */
                        err = check_reg_arg(env, insn->dst_reg, SRC_OP);
                        if (err)
                                return err;

                        dst_reg_type = regs[insn->dst_reg].type;

                        /* check that memory (dst_reg + off) is writeable */
                        err = check_mem_access(env, env->insn_idx, insn->dst_reg,
                                               insn->off, BPF_SIZE(insn->code),
                                               BPF_WRITE, insn->src_reg, false, false);
                        if (err)
                                return err;

                        err = save_aux_ptr_type(env, dst_reg_type, false);
                        if (err)
                                return err;
                } else if (class == BPF_ST) {
                        enum bpf_reg_type dst_reg_type;

                        if (BPF_MODE(insn->code) != BPF_MEM ||
                            insn->src_reg != BPF_REG_0) {
                                verbose(env, "BPF_ST uses reserved fields\n");
                                return -EINVAL;
                        }
                        /* check src operand */
                        err = check_reg_arg(env, insn->dst_reg, SRC_OP);
                        if (err)
                                return err;

                        dst_reg_type = regs[insn->dst_reg].type;

                        /* check that memory (dst_reg + off) is writeable */
                        err = check_mem_access(env, env->insn_idx, insn->dst_reg,
                                               insn->off, BPF_SIZE(insn->code),
                                               BPF_WRITE, -1, false, false);
                        if (err)
                                return err;

                        err = save_aux_ptr_type(env, dst_reg_type, false);
                        if (err)
                                return err;
                } else if (class == BPF_JMP || class == BPF_JMP32) {
                        u8 opcode = BPF_OP(insn->code);

                        env->jmps_processed++;
                        if (opcode == BPF_CALL) {
                                if (BPF_SRC(insn->code) != BPF_K ||
                                    (insn->src_reg != BPF_PSEUDO_KFUNC_CALL
                                     && insn->off != 0) ||
                                    (insn->src_reg != BPF_REG_0 &&
                                     insn->src_reg != BPF_PSEUDO_CALL &&
                                     insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
                                    insn->dst_reg != BPF_REG_0 ||
                                    class == BPF_JMP32) {
                                        verbose(env, "BPF_CALL uses reserved fields\n");
                                        return -EINVAL;
                                }

                                if (env->cur_state->active_locks) {
                                        if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) ||
                                            (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
                                             (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) {
                                                verbose(env, "function calls are not allowed while holding a lock\n");
                                                return -EINVAL;
                                        }
                                }
                                if (insn->src_reg == BPF_PSEUDO_CALL) {
                                        err = check_func_call(env, insn, &env->insn_idx);
                                } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
                                        err = check_kfunc_call(env, insn, &env->insn_idx);
                                        if (!err && is_bpf_throw_kfunc(insn)) {
                                                exception_exit = true;
                                                goto process_bpf_exit_full;
                                        }
                                } else {
                                        err = check_helper_call(env, insn, &env->insn_idx);
                                }
                                if (err)
                                        return err;

                                mark_reg_scratched(env, BPF_REG_0);
                        } else if (opcode == BPF_JA) {
                                if (BPF_SRC(insn->code) != BPF_K ||
                                    insn->src_reg != BPF_REG_0 ||
                                    insn->dst_reg != BPF_REG_0 ||
                                    (class == BPF_JMP && insn->imm != 0) ||
                                    (class == BPF_JMP32 && insn->off != 0)) {
                                        verbose(env, "BPF_JA uses reserved fields\n");
                                        return -EINVAL;
                                }

                                if (class == BPF_JMP)
                                        env->insn_idx += insn->off + 1;
                                else
                                        env->insn_idx += insn->imm + 1;
                                continue;

                        } else if (opcode == BPF_EXIT) {
                                if (BPF_SRC(insn->code) != BPF_K ||
                                    insn->imm != 0 ||
                                    insn->src_reg != BPF_REG_0 ||
                                    insn->dst_reg != BPF_REG_0 ||
                                    class == BPF_JMP32) {
                                        verbose(env, "BPF_EXIT uses reserved fields\n");
                                        return -EINVAL;
                                }
process_bpf_exit_full:
                                /* We must do check_reference_leak here before
                                 * prepare_func_exit to handle the case when
                                 * state->curframe > 0, it may be a callback
                                 * function, for which reference_state must
                                 * match caller reference state when it exits.
                                 */
                                err = check_resource_leak(env, exception_exit, !env->cur_state->curframe,
                                                          "BPF_EXIT instruction in main prog");
                                if (err)
                                        return err;

                                /* The side effect of the prepare_func_exit
                                 * which is being skipped is that it frees
                                 * bpf_func_state. Typically, process_bpf_exit
                                 * will only be hit with outermost exit.
                                 * copy_verifier_state in pop_stack will handle
                                 * freeing of any extra bpf_func_state left over
                                 * from not processing all nested function
                                 * exits. We also skip return code checks as
                                 * they are not needed for exceptional exits.
                                 */
                                if (exception_exit)
                                        goto process_bpf_exit;

                                if (state->curframe) {
                                        /* exit from nested function */
                                        err = prepare_func_exit(env, &env->insn_idx);
                                        if (err)
                                                return err;
                                        do_print_state = true;
                                        continue;
                                }

                                err = check_return_code(env, BPF_REG_0, "R0");
                                if (err)
                                        return err;
process_bpf_exit:
                                mark_verifier_state_scratched(env);
                                update_branch_counts(env, env->cur_state);
                                err = pop_stack(env, &prev_insn_idx,
                                                &env->insn_idx, pop_log);
                                if (err < 0) {
                                        if (err != -ENOENT)
                                                return err;
                                        break;
                                } else {
                                        do_print_state = true;
                                        continue;
                                }
                        } else {
                                err = check_cond_jmp_op(env, insn, &env->insn_idx);
                                if (err)
                                        return err;
                        }
                } else if (class == BPF_LD) {
                        u8 mode = BPF_MODE(insn->code);

                        if (mode == BPF_ABS || mode == BPF_IND) {
                                err = check_ld_abs(env, insn);
                                if (err)
                                        return err;

                        } else if (mode == BPF_IMM) {
                                err = check_ld_imm(env, insn);
                                if (err)
                                        return err;

                                env->insn_idx++;
                                sanitize_mark_insn_seen(env);
                        } else {
                                verbose(env, "invalid BPF_LD mode\n");
                                return -EINVAL;
                        }
                } else {
                        verbose(env, "unknown insn class %d\n", class);
                        return -EINVAL;
                }

                env->insn_idx++;
        }

        return 0;
}

static int find_btf_percpu_datasec(struct btf *btf)
{
        const struct btf_type *t;
        const char *tname;
        int i, n;

        /*
         * Both vmlinux and module each have their own ".data..percpu"
         * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF
         * types to look at only module's own BTF types.
         */
        n = btf_nr_types(btf);
        if (btf_is_module(btf))
                i = btf_nr_types(btf_vmlinux);
        else
                i = 1;

        for(; i < n; i++) {
                t = btf_type_by_id(btf, i);
                if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC)
                        continue;

                tname = btf_name_by_offset(btf, t->name_off);
                if (!strcmp(tname, ".data..percpu"))
                        return i;
        }

        return -ENOENT;
}

/*
 * Add btf to the used_btfs array and return the index. (If the btf was
 * already added, then just return the index.) Upon successful insertion
 * increase btf refcnt, and, if present, also refcount the corresponding
 * kernel module.
 */
static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf)
{
        struct btf_mod_pair *btf_mod;
        int i;

        /* check whether we recorded this BTF (and maybe module) already */
        for (i = 0; i < env->used_btf_cnt; i++)
                if (env->used_btfs[i].btf == btf)
                        return i;

        if (env->used_btf_cnt >= MAX_USED_BTFS)
                return -E2BIG;

        btf_get(btf);

        btf_mod = &env->used_btfs[env->used_btf_cnt];
        btf_mod->btf = btf;
        btf_mod->module = NULL;

        /* if we reference variables from kernel module, bump its refcount */
        if (btf_is_module(btf)) {
                btf_mod->module = btf_try_get_module(btf);
                if (!btf_mod->module) {
                        btf_put(btf);
                        return -ENXIO;
                }
        }

        return env->used_btf_cnt++;
}

/* replace pseudo btf_id with kernel symbol address */
static int __check_pseudo_btf_id(struct bpf_verifier_env *env,
                                 struct bpf_insn *insn,
                                 struct bpf_insn_aux_data *aux,
                                 struct btf *btf)
{
        const struct btf_var_secinfo *vsi;
        const struct btf_type *datasec;
        const struct btf_type *t;
        const char *sym_name;
        bool percpu = false;
        u32 type, id = insn->imm;
        s32 datasec_id;
        u64 addr;
        int i;

        t = btf_type_by_id(btf, id);
        if (!t) {
                verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
                return -ENOENT;
        }

        if (!btf_type_is_var(t) && !btf_type_is_func(t)) {
                verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id);
                return -EINVAL;
        }

        sym_name = btf_name_by_offset(btf, t->name_off);
        addr = kallsyms_lookup_name(sym_name);
        if (!addr) {
                verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
                        sym_name);
                return -ENOENT;
        }
        insn[0].imm = (u32)addr;
        insn[1].imm = addr >> 32;

        if (btf_type_is_func(t)) {
                aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
                aux->btf_var.mem_size = 0;
                return 0;
        }

        datasec_id = find_btf_percpu_datasec(btf);
        if (datasec_id > 0) {
                datasec = btf_type_by_id(btf, datasec_id);
                for_each_vsi(i, datasec, vsi) {
                        if (vsi->type == id) {
                                percpu = true;
                                break;
                        }
                }
        }

        type = t->type;
        t = btf_type_skip_modifiers(btf, type, NULL);
        if (percpu) {
                aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU;
                aux->btf_var.btf = btf;
                aux->btf_var.btf_id = type;
        } else if (!btf_type_is_struct(t)) {
                const struct btf_type *ret;
                const char *tname;
                u32 tsize;

                /* resolve the type size of ksym. */
                ret = btf_resolve_size(btf, t, &tsize);
                if (IS_ERR(ret)) {
                        tname = btf_name_by_offset(btf, t->name_off);
                        verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
                                tname, PTR_ERR(ret));
                        return -EINVAL;
                }
                aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
                aux->btf_var.mem_size = tsize;
        } else {
                aux->btf_var.reg_type = PTR_TO_BTF_ID;
                aux->btf_var.btf = btf;
                aux->btf_var.btf_id = type;
        }

        return 0;
}

static int check_pseudo_btf_id(struct bpf_verifier_env *env,
                               struct bpf_insn *insn,
                               struct bpf_insn_aux_data *aux)
{
        struct btf *btf;
        int btf_fd;
        int err;

        btf_fd = insn[1].imm;
        if (btf_fd) {
                CLASS(fd, f)(btf_fd);

                btf = __btf_get_by_fd(f);
                if (IS_ERR(btf)) {
                        verbose(env, "invalid module BTF object FD specified.\n");
                        return -EINVAL;
                }
        } else {
                if (!btf_vmlinux) {
                        verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
                        return -EINVAL;
                }
                btf = btf_vmlinux;
        }

        err = __check_pseudo_btf_id(env, insn, aux, btf);
        if (err)
                return err;

        err = __add_used_btf(env, btf);
        if (err < 0)
                return err;
        return 0;
}

static bool is_tracing_prog_type(enum bpf_prog_type type)
{
        switch (type) {
        case BPF_PROG_TYPE_KPROBE:
        case BPF_PROG_TYPE_TRACEPOINT:
        case BPF_PROG_TYPE_PERF_EVENT:
        case BPF_PROG_TYPE_RAW_TRACEPOINT:
        case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
                return true;
        default:
                return false;
        }
}

static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
{
        return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
                map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
}

static int check_map_prog_compatibility(struct bpf_verifier_env *env,
                                        struct bpf_map *map,
                                        struct bpf_prog *prog)

{
        enum bpf_prog_type prog_type = resolve_prog_type(prog);

        if (btf_record_has_field(map->record, BPF_LIST_HEAD) ||
            btf_record_has_field(map->record, BPF_RB_ROOT)) {
                if (is_tracing_prog_type(prog_type)) {
                        verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n");
                        return -EINVAL;
                }
        }

        if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
                if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
                        verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
                        return -EINVAL;
                }

                if (is_tracing_prog_type(prog_type)) {
                        verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
                        return -EINVAL;
                }
        }

        if (btf_record_has_field(map->record, BPF_TIMER)) {
                if (is_tracing_prog_type(prog_type)) {
                        verbose(env, "tracing progs cannot use bpf_timer yet\n");
                        return -EINVAL;
                }
        }

        if (btf_record_has_field(map->record, BPF_WORKQUEUE)) {
                if (is_tracing_prog_type(prog_type)) {
                        verbose(env, "tracing progs cannot use bpf_wq yet\n");
                        return -EINVAL;
                }
        }

        if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) &&
            !bpf_offload_prog_map_match(prog, map)) {
                verbose(env, "offload device mismatch between prog and map\n");
                return -EINVAL;
        }

        if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
                verbose(env, "bpf_struct_ops map cannot be used in prog\n");
                return -EINVAL;
        }

        if (prog->sleepable)
                switch (map->map_type) {
                case BPF_MAP_TYPE_HASH:
                case BPF_MAP_TYPE_LRU_HASH:
                case BPF_MAP_TYPE_ARRAY:
                case BPF_MAP_TYPE_PERCPU_HASH:
                case BPF_MAP_TYPE_PERCPU_ARRAY:
                case BPF_MAP_TYPE_LRU_PERCPU_HASH:
                case BPF_MAP_TYPE_ARRAY_OF_MAPS:
                case BPF_MAP_TYPE_HASH_OF_MAPS:
                case BPF_MAP_TYPE_RINGBUF:
                case BPF_MAP_TYPE_USER_RINGBUF:
                case BPF_MAP_TYPE_INODE_STORAGE:
                case BPF_MAP_TYPE_SK_STORAGE:
                case BPF_MAP_TYPE_TASK_STORAGE:
                case BPF_MAP_TYPE_CGRP_STORAGE:
                case BPF_MAP_TYPE_QUEUE:
                case BPF_MAP_TYPE_STACK:
                case BPF_MAP_TYPE_ARENA:
                        break;
                default:
                        verbose(env,
                                "Sleepable programs can only use array, hash, ringbuf and local storage maps\n");
                        return -EINVAL;
                }

        if (bpf_map_is_cgroup_storage(map) &&
            bpf_cgroup_storage_assign(env->prog->aux, map)) {
                verbose(env, "only one cgroup storage of each type is allowed\n");
                return -EBUSY;
        }

        if (map->map_type == BPF_MAP_TYPE_ARENA) {
                if (env->prog->aux->arena) {
                        verbose(env, "Only one arena per program\n");
                        return -EBUSY;
                }
                if (!env->allow_ptr_leaks || !env->bpf_capable) {
                        verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n");
                        return -EPERM;
                }
                if (!env->prog->jit_requested) {
                        verbose(env, "JIT is required to use arena\n");
                        return -EOPNOTSUPP;
                }
                if (!bpf_jit_supports_arena()) {
                        verbose(env, "JIT doesn't support arena\n");
                        return -EOPNOTSUPP;
                }
                env->prog->aux->arena = (void *)map;
                if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) {
                        verbose(env, "arena's user address must be set via map_extra or mmap()\n");
                        return -EINVAL;
                }
        }

        return 0;
}

static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)
{
        int i, err;

        /* check whether we recorded this map already */
        for (i = 0; i < env->used_map_cnt; i++)
                if (env->used_maps[i] == map)
                        return i;

        if (env->used_map_cnt >= MAX_USED_MAPS) {
                verbose(env, "The total number of maps per program has reached the limit of %u\n",
                        MAX_USED_MAPS);
                return -E2BIG;
        }

        err = check_map_prog_compatibility(env, map, env->prog);
        if (err)
                return err;

        if (env->prog->sleepable)
                atomic64_inc(&map->sleepable_refcnt);

        /* hold the map. If the program is rejected by verifier,
         * the map will be released by release_maps() or it
         * will be used by the valid program until it's unloaded
         * and all maps are released in bpf_free_used_maps()
         */
        bpf_map_inc(map);

        env->used_maps[env->used_map_cnt++] = map;

        return env->used_map_cnt - 1;
}

/* Add map behind fd to used maps list, if it's not already there, and return
 * its index.
 * Returns <0 on error, or >= 0 index, on success.
 */
static int add_used_map(struct bpf_verifier_env *env, int fd)
{
        struct bpf_map *map;
        CLASS(fd, f)(fd);

        map = __bpf_map_get(f);
        if (IS_ERR(map)) {
                verbose(env, "fd %d is not pointing to valid bpf_map\n", fd);
                return PTR_ERR(map);
        }

        return __add_used_map(env, map);
}

/* find and rewrite pseudo imm in ld_imm64 instructions:
 *
 * 1. if it accesses map FD, replace it with actual map pointer.
 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
 *
 * NOTE: btf_vmlinux is required for converting pseudo btf_id.
 */
static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
{
        struct bpf_insn *insn = env->prog->insnsi;
        int insn_cnt = env->prog->len;
        int i, err;

        err = bpf_prog_calc_tag(env->prog);
        if (err)
                return err;

        for (i = 0; i < insn_cnt; i++, insn++) {
                if (BPF_CLASS(insn->code) == BPF_LDX &&
                    ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) ||
                    insn->imm != 0)) {
                        verbose(env, "BPF_LDX uses reserved fields\n");
                        return -EINVAL;
                }

                if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
                        struct bpf_insn_aux_data *aux;
                        struct bpf_map *map;
                        int map_idx;
                        u64 addr;
                        u32 fd;

                        if (i == insn_cnt - 1 || insn[1].code != 0 ||
                            insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
                            insn[1].off != 0) {
                                verbose(env, "invalid bpf_ld_imm64 insn\n");
                                return -EINVAL;
                        }

                        if (insn[0].src_reg == 0)
                                /* valid generic load 64-bit imm */
                                goto next_insn;

                        if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
                                aux = &env->insn_aux_data[i];
                                err = check_pseudo_btf_id(env, insn, aux);
                                if (err)
                                        return err;
                                goto next_insn;
                        }

                        if (insn[0].src_reg == BPF_PSEUDO_FUNC) {
                                aux = &env->insn_aux_data[i];
                                aux->ptr_type = PTR_TO_FUNC;
                                goto next_insn;
                        }

                        /* In final convert_pseudo_ld_imm64() step, this is
                         * converted into regular 64-bit imm load insn.
                         */
                        switch (insn[0].src_reg) {
                        case BPF_PSEUDO_MAP_VALUE:
                        case BPF_PSEUDO_MAP_IDX_VALUE:
                                break;
                        case BPF_PSEUDO_MAP_FD:
                        case BPF_PSEUDO_MAP_IDX:
                                if (insn[1].imm == 0)
                                        break;
                                fallthrough;
                        default:
                                verbose(env, "unrecognized bpf_ld_imm64 insn\n");
                                return -EINVAL;
                        }

                        switch (insn[0].src_reg) {
                        case BPF_PSEUDO_MAP_IDX_VALUE:
                        case BPF_PSEUDO_MAP_IDX:
                                if (bpfptr_is_null(env->fd_array)) {
                                        verbose(env, "fd_idx without fd_array is invalid\n");
                                        return -EPROTO;
                                }
                                if (copy_from_bpfptr_offset(&fd, env->fd_array,
                                                            insn[0].imm * sizeof(fd),
                                                            sizeof(fd)))
                                        return -EFAULT;
                                break;
                        default:
                                fd = insn[0].imm;
                                break;
                        }

                        map_idx = add_used_map(env, fd);
                        if (map_idx < 0)
                                return map_idx;
                        map = env->used_maps[map_idx];

                        aux = &env->insn_aux_data[i];
                        aux->map_index = map_idx;

                        if (insn[0].src_reg == BPF_PSEUDO_MAP_FD ||
                            insn[0].src_reg == BPF_PSEUDO_MAP_IDX) {
                                addr = (unsigned long)map;
                        } else {
                                u32 off = insn[1].imm;

                                if (off >= BPF_MAX_VAR_OFF) {
                                        verbose(env, "direct value offset of %u is not allowed\n", off);
                                        return -EINVAL;
                                }

                                if (!map->ops->map_direct_value_addr) {
                                        verbose(env, "no direct value access support for this map type\n");
                                        return -EINVAL;
                                }

                                err = map->ops->map_direct_value_addr(map, &addr, off);
                                if (err) {
                                        verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
                                                map->value_size, off);
                                        return err;
                                }

                                aux->map_off = off;
                                addr += off;
                        }

                        insn[0].imm = (u32)addr;
                        insn[1].imm = addr >> 32;

next_insn:
                        insn++;
                        i++;
                        continue;
                }

                /* Basic sanity check before we invest more work here. */
                if (!bpf_opcode_in_insntable(insn->code)) {
                        verbose(env, "unknown opcode %02x\n", insn->code);
                        return -EINVAL;
                }
        }

        /* now all pseudo BPF_LD_IMM64 instructions load valid
         * 'struct bpf_map *' into a register instead of user map_fd.
         * These pointers will be used later by verifier to validate map access.
         */
        return 0;
}

/* drop refcnt of maps used by the rejected program */
static void release_maps(struct bpf_verifier_env *env)
{
        __bpf_free_used_maps(env->prog->aux, env->used_maps,
                             env->used_map_cnt);
}

/* drop refcnt of maps used by the rejected program */
static void release_btfs(struct bpf_verifier_env *env)
{
        __bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt);
}

/* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
{
        struct bpf_insn *insn = env->prog->insnsi;
        int insn_cnt = env->prog->len;
        int i;

        for (i = 0; i < insn_cnt; i++, insn++) {
                if (insn->code != (BPF_LD | BPF_IMM | BPF_DW))
                        continue;
                if (insn->src_reg == BPF_PSEUDO_FUNC)
                        continue;
                insn->src_reg = 0;
        }
}

/* single env->prog->insni[off] instruction was replaced with the range
 * insni[off, off + cnt).  Adjust corresponding insn_aux_data by copying
 * [0, off) and [off, end) to new locations, so the patched range stays zero
 */
static void adjust_insn_aux_data(struct bpf_verifier_env *env,
                                 struct bpf_insn_aux_data *new_data,
                                 struct bpf_prog *new_prog, u32 off, u32 cnt)
{
        struct bpf_insn_aux_data *old_data = env->insn_aux_data;
        struct bpf_insn *insn = new_prog->insnsi;
        u32 old_seen = old_data[off].seen;
        u32 prog_len;
        int i;

        /* aux info at OFF always needs adjustment, no matter fast path
         * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the
         * original insn at old prog.
         */
        old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);

        if (cnt == 1)
                return;
        prog_len = new_prog->len;

        memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
        memcpy(new_data + off + cnt - 1, old_data + off,
               sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
        for (i = off; i < off + cnt - 1; i++) {
                /* Expand insni[off]'s seen count to the patched range. */
                new_data[i].seen = old_seen;
                new_data[i].zext_dst = insn_has_def32(env, insn + i);
        }
        env->insn_aux_data = new_data;
        vfree(old_data);
}

static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
{
        int i;

        if (len == 1)
                return;
        /* NOTE: fake 'exit' subprog should be updated as well. */
        for (i = 0; i <= env->subprog_cnt; i++) {
                if (env->subprog_info[i].start <= off)
                        continue;
                env->subprog_info[i].start += len - 1;
        }
}

static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
{
        struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
        int i, sz = prog->aux->size_poke_tab;
        struct bpf_jit_poke_descriptor *desc;

        for (i = 0; i < sz; i++) {
                desc = &tab[i];
                if (desc->insn_idx <= off)
                        continue;
                desc->insn_idx += len - 1;
        }
}

static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
                                            const struct bpf_insn *patch, u32 len)
{
        struct bpf_prog *new_prog;
        struct bpf_insn_aux_data *new_data = NULL;

        if (len > 1) {
                new_data = vzalloc(array_size(env->prog->len + len - 1,
                                              sizeof(struct bpf_insn_aux_data)));
                if (!new_data)
                        return NULL;
        }

        new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
        if (IS_ERR(new_prog)) {
                if (PTR_ERR(new_prog) == -ERANGE)
                        verbose(env,
                                "insn %d cannot be patched due to 16-bit range\n",
                                env->insn_aux_data[off].orig_idx);
                vfree(new_data);
                return NULL;
        }
        adjust_insn_aux_data(env, new_data, new_prog, off, len);
        adjust_subprog_starts(env, off, len);
        adjust_poke_descs(new_prog, off, len);
        return new_prog;
}

/*
 * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the
 * jump offset by 'delta'.
 */
static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta)
{
        struct bpf_insn *insn = prog->insnsi;
        u32 insn_cnt = prog->len, i;
        s32 imm;
        s16 off;

        for (i = 0; i < insn_cnt; i++, insn++) {
                u8 code = insn->code;

                if (tgt_idx <= i && i < tgt_idx + delta)
                        continue;

                if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) ||
                    BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT)
                        continue;

                if (insn->code == (BPF_JMP32 | BPF_JA)) {
                        if (i + 1 + insn->imm != tgt_idx)
                                continue;
                        if (check_add_overflow(insn->imm, delta, &imm))
                                return -ERANGE;
                        insn->imm = imm;
                } else {
                        if (i + 1 + insn->off != tgt_idx)
                                continue;
                        if (check_add_overflow(insn->off, delta, &off))
                                return -ERANGE;
                        insn->off = off;
                }
        }
        return 0;
}

static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
                                              u32 off, u32 cnt)
{
        int i, j;

        /* find first prog starting at or after off (first to remove) */
        for (i = 0; i < env->subprog_cnt; i++)
                if (env->subprog_info[i].start >= off)
                        break;
        /* find first prog starting at or after off + cnt (first to stay) */
        for (j = i; j < env->subprog_cnt; j++)
                if (env->subprog_info[j].start >= off + cnt)
                        break;
        /* if j doesn't start exactly at off + cnt, we are just removing
         * the front of previous prog
         */
        if (env->subprog_info[j].start != off + cnt)
                j--;

        if (j > i) {
                struct bpf_prog_aux *aux = env->prog->aux;
                int move;

                /* move fake 'exit' subprog as well */
                move = env->subprog_cnt + 1 - j;

                memmove(env->subprog_info + i,
                        env->subprog_info + j,
                        sizeof(*env->subprog_info) * move);
                env->subprog_cnt -= j - i;

                /* remove func_info */
                if (aux->func_info) {
                        move = aux->func_info_cnt - j;

                        memmove(aux->func_info + i,
                                aux->func_info + j,
                                sizeof(*aux->func_info) * move);
                        aux->func_info_cnt -= j - i;
                        /* func_info->insn_off is set after all code rewrites,
                         * in adjust_btf_func() - no need to adjust
                         */
                }
        } else {
                /* convert i from "first prog to remove" to "first to adjust" */
                if (env->subprog_info[i].start == off)
                        i++;
        }

        /* update fake 'exit' subprog as well */
        for (; i <= env->subprog_cnt; i++)
                env->subprog_info[i].start -= cnt;

        return 0;
}

static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
                                      u32 cnt)
{
        struct bpf_prog *prog = env->prog;
        u32 i, l_off, l_cnt, nr_linfo;
        struct bpf_line_info *linfo;

        nr_linfo = prog->aux->nr_linfo;
        if (!nr_linfo)
                return 0;

        linfo = prog->aux->linfo;

        /* find first line info to remove, count lines to be removed */
        for (i = 0; i < nr_linfo; i++)
                if (linfo[i].insn_off >= off)
                        break;

        l_off = i;
        l_cnt = 0;
        for (; i < nr_linfo; i++)
                if (linfo[i].insn_off < off + cnt)
                        l_cnt++;
                else
                        break;

        /* First live insn doesn't match first live linfo, it needs to "inherit"
         * last removed linfo.  prog is already modified, so prog->len == off
         * means no live instructions after (tail of the program was removed).
         */
        if (prog->len != off && l_cnt &&
            (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
                l_cnt--;
                linfo[--i].insn_off = off + cnt;
        }

        /* remove the line info which refer to the removed instructions */
        if (l_cnt) {
                memmove(linfo + l_off, linfo + i,
                        sizeof(*linfo) * (nr_linfo - i));

                prog->aux->nr_linfo -= l_cnt;
                nr_linfo = prog->aux->nr_linfo;
        }

        /* pull all linfo[i].insn_off >= off + cnt in by cnt */
        for (i = l_off; i < nr_linfo; i++)
                linfo[i].insn_off -= cnt;

        /* fix up all subprogs (incl. 'exit') which start >= off */
        for (i = 0; i <= env->subprog_cnt; i++)
                if (env->subprog_info[i].linfo_idx > l_off) {
                        /* program may have started in the removed region but
                         * may not be fully removed
                         */
                        if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
                                env->subprog_info[i].linfo_idx -= l_cnt;
                        else
                                env->subprog_info[i].linfo_idx = l_off;
                }

        return 0;
}

static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
{
        struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
        unsigned int orig_prog_len = env->prog->len;
        int err;

        if (bpf_prog_is_offloaded(env->prog->aux))
                bpf_prog_offload_remove_insns(env, off, cnt);

        err = bpf_remove_insns(env->prog, off, cnt);
        if (err)
                return err;

        err = adjust_subprog_starts_after_remove(env, off, cnt);
        if (err)
                return err;

        err = bpf_adj_linfo_after_remove(env, off, cnt);
        if (err)
                return err;

        memmove(aux_data + off,        aux_data + off + cnt,
                sizeof(*aux_data) * (orig_prog_len - off - cnt));

        return 0;
}

/* The verifier does more data flow analysis than llvm and will not
 * explore branches that are dead at run time. Malicious programs can
 * have dead code too. Therefore replace all dead at-run-time code
 * with 'ja -1'.
 *
 * Just nops are not optimal, e.g. if they would sit at the end of the
 * program and through another bug we would manage to jump there, then
 * we'd execute beyond program memory otherwise. Returning exception
 * code also wouldn't work since we can have subprogs where the dead
 * code could be located.
 */
static void sanitize_dead_code(struct bpf_verifier_env *env)
{
        struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
        struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
        struct bpf_insn *insn = env->prog->insnsi;
        const int insn_cnt = env->prog->len;
        int i;

        for (i = 0; i < insn_cnt; i++) {
                if (aux_data[i].seen)
                        continue;
                memcpy(insn + i, &trap, sizeof(trap));
                aux_data[i].zext_dst = false;
        }
}

static bool insn_is_cond_jump(u8 code)
{
        u8 op;

        op = BPF_OP(code);
        if (BPF_CLASS(code) == BPF_JMP32)
                return op != BPF_JA;

        if (BPF_CLASS(code) != BPF_JMP)
                return false;

        return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
}

static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
{
        struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
        struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
        struct bpf_insn *insn = env->prog->insnsi;
        const int insn_cnt = env->prog->len;
        int i;

        for (i = 0; i < insn_cnt; i++, insn++) {
                if (!insn_is_cond_jump(insn->code))
                        continue;

                if (!aux_data[i + 1].seen)
                        ja.off = insn->off;
                else if (!aux_data[i + 1 + insn->off].seen)
                        ja.off = 0;
                else
                        continue;

                if (bpf_prog_is_offloaded(env->prog->aux))
                        bpf_prog_offload_replace_insn(env, i, &ja);

                memcpy(insn, &ja, sizeof(ja));
        }
}

static int opt_remove_dead_code(struct bpf_verifier_env *env)
{
        struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
        int insn_cnt = env->prog->len;
        int i, err;

        for (i = 0; i < insn_cnt; i++) {
                int j;

                j = 0;
                while (i + j < insn_cnt && !aux_data[i + j].seen)
                        j++;
                if (!j)
                        continue;

                err = verifier_remove_insns(env, i, j);
                if (err)
                        return err;
                insn_cnt = env->prog->len;
        }

        return 0;
}

static const struct bpf_insn NOP = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
static const struct bpf_insn MAY_GOTO_0 = BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0, 0);

static int opt_remove_nops(struct bpf_verifier_env *env)
{
        struct bpf_insn *insn = env->prog->insnsi;
        int insn_cnt = env->prog->len;
        bool is_may_goto_0, is_ja;
        int i, err;

        for (i = 0; i < insn_cnt; i++) {
                is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0));
                is_ja = !memcmp(&insn[i], &NOP, sizeof(NOP));

                if (!is_may_goto_0 && !is_ja)
                        continue;

                err = verifier_remove_insns(env, i, 1);
                if (err)
                        return err;
                insn_cnt--;
                /* Go back one insn to catch may_goto +1; may_goto +0 sequence */
                i -= (is_may_goto_0 && i > 0) ? 2 : 1;
        }

        return 0;
}

static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
                                         const union bpf_attr *attr)
{
        struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4];
        struct bpf_insn_aux_data *aux = env->insn_aux_data;
        int i, patch_len, delta = 0, len = env->prog->len;
        struct bpf_insn *insns = env->prog->insnsi;
        struct bpf_prog *new_prog;
        bool rnd_hi32;

        rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32;
        zext_patch[1] = BPF_ZEXT_REG(0);
        rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0);
        rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
        rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX);
        for (i = 0; i < len; i++) {
                int adj_idx = i + delta;
                struct bpf_insn insn;
                int load_reg;

                insn = insns[adj_idx];
                load_reg = insn_def_regno(&insn);
                if (!aux[adj_idx].zext_dst) {
                        u8 code, class;
                        u32 imm_rnd;

                        if (!rnd_hi32)
                                continue;

                        code = insn.code;
                        class = BPF_CLASS(code);
                        if (load_reg == -1)
                                continue;

                        /* NOTE: arg "reg" (the fourth one) is only used for
                         *       BPF_STX + SRC_OP, so it is safe to pass NULL
                         *       here.
                         */
                        if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {
                                if (class == BPF_LD &&
                                    BPF_MODE(code) == BPF_IMM)
                                        i++;
                                continue;
                        }

                        /* ctx load could be transformed into wider load. */
                        if (class == BPF_LDX &&
                            aux[adj_idx].ptr_type == PTR_TO_CTX)
                                continue;

                        imm_rnd = get_random_u32();
                        rnd_hi32_patch[0] = insn;
                        rnd_hi32_patch[1].imm = imm_rnd;
                        rnd_hi32_patch[3].dst_reg = load_reg;
                        patch = rnd_hi32_patch;
                        patch_len = 4;
                        goto apply_patch_buffer;
                }

                /* Add in an zero-extend instruction if a) the JIT has requested
                 * it or b) it's a CMPXCHG.
                 *
                 * The latter is because: BPF_CMPXCHG always loads a value into
                 * R0, therefore always zero-extends. However some archs'
                 * equivalent instruction only does this load when the
                 * comparison is successful. This detail of CMPXCHG is
                 * orthogonal to the general zero-extension behaviour of the
                 * CPU, so it's treated independently of bpf_jit_needs_zext.
                 */
                if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn))
                        continue;

                /* Zero-extension is done by the caller. */
                if (bpf_pseudo_kfunc_call(&insn))
                        continue;

                if (WARN_ON(load_reg == -1)) {
                        verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");
                        return -EFAULT;
                }

                zext_patch[0] = insn;
                zext_patch[1].dst_reg = load_reg;
                zext_patch[1].src_reg = load_reg;
                patch = zext_patch;
                patch_len = 2;
apply_patch_buffer:
                new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);
                if (!new_prog)
                        return -ENOMEM;
                env->prog = new_prog;
                insns = new_prog->insnsi;
                aux = env->insn_aux_data;
                delta += patch_len - 1;
        }

        return 0;
}

/* convert load instructions that access fields of a context type into a
 * sequence of instructions that access fields of the underlying structure:
 *     struct __sk_buff    -> struct sk_buff
 *     struct bpf_sock_ops -> struct sock
 */
static int convert_ctx_accesses(struct bpf_verifier_env *env)
{
        struct bpf_subprog_info *subprogs = env->subprog_info;
        const struct bpf_verifier_ops *ops = env->ops;
        int i, cnt, size, ctx_field_size, delta = 0, epilogue_cnt = 0;
        const int insn_cnt = env->prog->len;
        struct bpf_insn *epilogue_buf = env->epilogue_buf;
        struct bpf_insn *insn_buf = env->insn_buf;
        struct bpf_insn *insn;
        u32 target_size, size_default, off;
        struct bpf_prog *new_prog;
        enum bpf_access_type type;
        bool is_narrower_load;
        int epilogue_idx = 0;

        if (ops->gen_epilogue) {
                epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,
                                                 -(subprogs[0].stack_depth + 8));
                if (epilogue_cnt >= INSN_BUF_SIZE) {
                        verbose(env, "bpf verifier is misconfigured\n");
                        return -EINVAL;
                } else if (epilogue_cnt) {
                        /* Save the ARG_PTR_TO_CTX for the epilogue to use */
                        cnt = 0;
                        subprogs[0].stack_depth += 8;
                        insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1,
                                                      -subprogs[0].stack_depth);
                        insn_buf[cnt++] = env->prog->insnsi[0];
                        new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;
                        env->prog = new_prog;
                        delta += cnt - 1;
                }
        }

        if (ops->gen_prologue || env->seen_direct_write) {
                if (!ops->gen_prologue) {
                        verbose(env, "bpf verifier is misconfigured\n");
                        return -EINVAL;
                }
                cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
                                        env->prog);
                if (cnt >= INSN_BUF_SIZE) {
                        verbose(env, "bpf verifier is misconfigured\n");
                        return -EINVAL;
                } else if (cnt) {
                        new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        env->prog = new_prog;
                        delta += cnt - 1;
                }
        }

        if (delta)
                WARN_ON(adjust_jmp_off(env->prog, 0, delta));

        if (bpf_prog_is_offloaded(env->prog->aux))
                return 0;

        insn = env->prog->insnsi + delta;

        for (i = 0; i < insn_cnt; i++, insn++) {
                bpf_convert_ctx_access_t convert_ctx_access;
                u8 mode;

                if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
                    insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
                    insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
                    insn->code == (BPF_LDX | BPF_MEM | BPF_DW) ||
                    insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) ||
                    insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) ||
                    insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) {
                        type = BPF_READ;
                } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
                           insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
                           insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
                           insn->code == (BPF_STX | BPF_MEM | BPF_DW) ||
                           insn->code == (BPF_ST | BPF_MEM | BPF_B) ||
                           insn->code == (BPF_ST | BPF_MEM | BPF_H) ||
                           insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
                           insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
                        type = BPF_WRITE;
                } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) ||
                            insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) &&
                           env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
                        insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code);
                        env->prog->aux->num_exentries++;
                        continue;
                } else if (insn->code == (BPF_JMP | BPF_EXIT) &&
                           epilogue_cnt &&
                           i + delta < subprogs[1].start) {
                        /* Generate epilogue for the main prog */
                        if (epilogue_idx) {
                                /* jump back to the earlier generated epilogue */
                                insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1);
                                cnt = 1;
                        } else {
                                memcpy(insn_buf, epilogue_buf,
                                       epilogue_cnt * sizeof(*epilogue_buf));
                                cnt = epilogue_cnt;
                                /* epilogue_idx cannot be 0. It must have at
                                 * least one ctx ptr saving insn before the
                                 * epilogue.
                                 */
                                epilogue_idx = i + delta;
                        }
                        goto patch_insn_buf;
                } else {
                        continue;
                }

                if (type == BPF_WRITE &&
                    env->insn_aux_data[i + delta].sanitize_stack_spill) {
                        struct bpf_insn patch[] = {
                                *insn,
                                BPF_ST_NOSPEC(),
                        };

                        cnt = ARRAY_SIZE(patch);
                        new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        continue;
                }

                switch ((int)env->insn_aux_data[i + delta].ptr_type) {
                case PTR_TO_CTX:
                        if (!ops->convert_ctx_access)
                                continue;
                        convert_ctx_access = ops->convert_ctx_access;
                        break;
                case PTR_TO_SOCKET:
                case PTR_TO_SOCK_COMMON:
                        convert_ctx_access = bpf_sock_convert_ctx_access;
                        break;
                case PTR_TO_TCP_SOCK:
                        convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
                        break;
                case PTR_TO_XDP_SOCK:
                        convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
                        break;
                case PTR_TO_BTF_ID:
                case PTR_TO_BTF_ID | PTR_UNTRUSTED:
                /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike
                 * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot
                 * be said once it is marked PTR_UNTRUSTED, hence we must handle
                 * any faults for loads into such types. BPF_WRITE is disallowed
                 * for this case.
                 */
                case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED:
                        if (type == BPF_READ) {
                                if (BPF_MODE(insn->code) == BPF_MEM)
                                        insn->code = BPF_LDX | BPF_PROBE_MEM |
                                                     BPF_SIZE((insn)->code);
                                else
                                        insn->code = BPF_LDX | BPF_PROBE_MEMSX |
                                                     BPF_SIZE((insn)->code);
                                env->prog->aux->num_exentries++;
                        }
                        continue;
                case PTR_TO_ARENA:
                        if (BPF_MODE(insn->code) == BPF_MEMSX) {
                                verbose(env, "sign extending loads from arena are not supported yet\n");
                                return -EOPNOTSUPP;
                        }
                        insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
                        env->prog->aux->num_exentries++;
                        continue;
                default:
                        continue;
                }

                ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
                size = BPF_LDST_BYTES(insn);
                mode = BPF_MODE(insn->code);

                /* If the read access is a narrower load of the field,
                 * convert to a 4/8-byte load, to minimum program type specific
                 * convert_ctx_access changes. If conversion is successful,
                 * we will apply proper mask to the result.
                 */
                is_narrower_load = size < ctx_field_size;
                size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
                off = insn->off;
                if (is_narrower_load) {
                        u8 size_code;

                        if (type == BPF_WRITE) {
                                verbose(env, "bpf verifier narrow ctx access misconfigured\n");
                                return -EINVAL;
                        }

                        size_code = BPF_H;
                        if (ctx_field_size == 4)
                                size_code = BPF_W;
                        else if (ctx_field_size == 8)
                                size_code = BPF_DW;

                        insn->off = off & ~(size_default - 1);
                        insn->code = BPF_LDX | BPF_MEM | size_code;
                }

                target_size = 0;
                cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
                                         &target_size);
                if (cnt == 0 || cnt >= INSN_BUF_SIZE ||
                    (ctx_field_size && !target_size)) {
                        verbose(env, "bpf verifier is misconfigured\n");
                        return -EINVAL;
                }

                if (is_narrower_load && size < target_size) {
                        u8 shift = bpf_ctx_narrow_access_offset(
                                off, size, size_default) * 8;
                        if (shift && cnt + 1 >= INSN_BUF_SIZE) {
                                verbose(env, "bpf verifier narrow ctx load misconfigured\n");
                                return -EINVAL;
                        }
                        if (ctx_field_size <= 4) {
                                if (shift)
                                        insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
                                                                        insn->dst_reg,
                                                                        shift);
                                insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
                                                                (1 << size * 8) - 1);
                        } else {
                                if (shift)
                                        insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
                                                                        insn->dst_reg,
                                                                        shift);
                                insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
                                                                (1ULL << size * 8) - 1);
                        }
                }
                if (mode == BPF_MEMSX)
                        insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X,
                                                       insn->dst_reg, insn->dst_reg,
                                                       size * 8, 0);

patch_insn_buf:
                new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                if (!new_prog)
                        return -ENOMEM;

                delta += cnt - 1;

                /* keep walking new program and skip insns we just inserted */
                env->prog = new_prog;
                insn      = new_prog->insnsi + i + delta;
        }

        return 0;
}

static int jit_subprogs(struct bpf_verifier_env *env)
{
        struct bpf_prog *prog = env->prog, **func, *tmp;
        int i, j, subprog_start, subprog_end = 0, len, subprog;
        struct bpf_map *map_ptr;
        struct bpf_insn *insn;
        void *old_bpf_func;
        int err, num_exentries;

        if (env->subprog_cnt <= 1)
                return 0;

        for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
                if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
                        continue;

                /* Upon error here we cannot fall back to interpreter but
                 * need a hard reject of the program. Thus -EFAULT is
                 * propagated in any case.
                 */
                subprog = find_subprog(env, i + insn->imm + 1);
                if (subprog < 0) {
                        WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
                                  i + insn->imm + 1);
                        return -EFAULT;
                }
                /* temporarily remember subprog id inside insn instead of
                 * aux_data, since next loop will split up all insns into funcs
                 */
                insn->off = subprog;
                /* remember original imm in case JIT fails and fallback
                 * to interpreter will be needed
                 */
                env->insn_aux_data[i].call_imm = insn->imm;
                /* point imm to __bpf_call_base+1 from JITs point of view */
                insn->imm = 1;
                if (bpf_pseudo_func(insn)) {
#if defined(MODULES_VADDR)
                        u64 addr = MODULES_VADDR;
#else
                        u64 addr = VMALLOC_START;
#endif
                        /* jit (e.g. x86_64) may emit fewer instructions
                         * if it learns a u32 imm is the same as a u64 imm.
                         * Set close enough to possible prog address.
                         */
                        insn[0].imm = (u32)addr;
                        insn[1].imm = addr >> 32;
                }
        }

        err = bpf_prog_alloc_jited_linfo(prog);
        if (err)
                goto out_undo_insn;

        err = -ENOMEM;
        func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
        if (!func)
                goto out_undo_insn;

        for (i = 0; i < env->subprog_cnt; i++) {
                subprog_start = subprog_end;
                subprog_end = env->subprog_info[i + 1].start;

                len = subprog_end - subprog_start;
                /* bpf_prog_run() doesn't call subprogs directly,
                 * hence main prog stats include the runtime of subprogs.
                 * subprogs don't have IDs and not reachable via prog_get_next_id
                 * func[i]->stats will never be accessed and stays NULL
                 */
                func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
                if (!func[i])
                        goto out_free;
                memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
                       len * sizeof(struct bpf_insn));
                func[i]->type = prog->type;
                func[i]->len = len;
                if (bpf_prog_calc_tag(func[i]))
                        goto out_free;
                func[i]->is_func = 1;
                func[i]->sleepable = prog->sleepable;
                func[i]->aux->func_idx = i;
                /* Below members will be freed only at prog->aux */
                func[i]->aux->btf = prog->aux->btf;
                func[i]->aux->func_info = prog->aux->func_info;
                func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
                func[i]->aux->poke_tab = prog->aux->poke_tab;
                func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;

                for (j = 0; j < prog->aux->size_poke_tab; j++) {
                        struct bpf_jit_poke_descriptor *poke;

                        poke = &prog->aux->poke_tab[j];
                        if (poke->insn_idx < subprog_end &&
                            poke->insn_idx >= subprog_start)
                                poke->aux = func[i]->aux;
                }

                func[i]->aux->name[0] = 'F';
                func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
                if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE)
                        func[i]->aux->jits_use_priv_stack = true;

                func[i]->jit_requested = 1;
                func[i]->blinding_requested = prog->blinding_requested;
                func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
                func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;
                func[i]->aux->linfo = prog->aux->linfo;
                func[i]->aux->nr_linfo = prog->aux->nr_linfo;
                func[i]->aux->jited_linfo = prog->aux->jited_linfo;
                func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
                func[i]->aux->arena = prog->aux->arena;
                num_exentries = 0;
                insn = func[i]->insnsi;
                for (j = 0; j < func[i]->len; j++, insn++) {
                        if (BPF_CLASS(insn->code) == BPF_LDX &&
                            (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
                             BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
                             BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
                                num_exentries++;
                        if ((BPF_CLASS(insn->code) == BPF_STX ||
                             BPF_CLASS(insn->code) == BPF_ST) &&
                             BPF_MODE(insn->code) == BPF_PROBE_MEM32)
                                num_exentries++;
                        if (BPF_CLASS(insn->code) == BPF_STX &&
                             BPF_MODE(insn->code) == BPF_PROBE_ATOMIC)
                                num_exentries++;
                }
                func[i]->aux->num_exentries = num_exentries;
                func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
                func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
                func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
                if (!i)
                        func[i]->aux->exception_boundary = env->seen_exception;
                func[i] = bpf_int_jit_compile(func[i]);
                if (!func[i]->jited) {
                        err = -ENOTSUPP;
                        goto out_free;
                }
                cond_resched();
        }

        /* at this point all bpf functions were successfully JITed
         * now populate all bpf_calls with correct addresses and
         * run last pass of JIT
         */
        for (i = 0; i < env->subprog_cnt; i++) {
                insn = func[i]->insnsi;
                for (j = 0; j < func[i]->len; j++, insn++) {
                        if (bpf_pseudo_func(insn)) {
                                subprog = insn->off;
                                insn[0].imm = (u32)(long)func[subprog]->bpf_func;
                                insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32;
                                continue;
                        }
                        if (!bpf_pseudo_call(insn))
                                continue;
                        subprog = insn->off;
                        insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func);
                }

                /* we use the aux data to keep a list of the start addresses
                 * of the JITed images for each function in the program
                 *
                 * for some architectures, such as powerpc64, the imm field
                 * might not be large enough to hold the offset of the start
                 * address of the callee's JITed image from __bpf_call_base
                 *
                 * in such cases, we can lookup the start address of a callee
                 * by using its subprog id, available from the off field of
                 * the call instruction, as an index for this list
                 */
                func[i]->aux->func = func;
                func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;
                func[i]->aux->real_func_cnt = env->subprog_cnt;
        }
        for (i = 0; i < env->subprog_cnt; i++) {
                old_bpf_func = func[i]->bpf_func;
                tmp = bpf_int_jit_compile(func[i]);
                if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
                        verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
                        err = -ENOTSUPP;
                        goto out_free;
                }
                cond_resched();
        }

        /* finally lock prog and jit images for all functions and
         * populate kallsysm. Begin at the first subprogram, since
         * bpf_prog_load will add the kallsyms for the main program.
         */
        for (i = 1; i < env->subprog_cnt; i++) {
                err = bpf_prog_lock_ro(func[i]);
                if (err)
                        goto out_free;
        }

        for (i = 1; i < env->subprog_cnt; i++)
                bpf_prog_kallsyms_add(func[i]);

        /* Last step: make now unused interpreter insns from main
         * prog consistent for later dump requests, so they can
         * later look the same as if they were interpreted only.
         */
        for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
                if (bpf_pseudo_func(insn)) {
                        insn[0].imm = env->insn_aux_data[i].call_imm;
                        insn[1].imm = insn->off;
                        insn->off = 0;
                        continue;
                }
                if (!bpf_pseudo_call(insn))
                        continue;
                insn->off = env->insn_aux_data[i].call_imm;
                subprog = find_subprog(env, i + insn->off + 1);
                insn->imm = subprog;
        }

        prog->jited = 1;
        prog->bpf_func = func[0]->bpf_func;
        prog->jited_len = func[0]->jited_len;
        prog->aux->extable = func[0]->aux->extable;
        prog->aux->num_exentries = func[0]->aux->num_exentries;
        prog->aux->func = func;
        prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt;
        prog->aux->real_func_cnt = env->subprog_cnt;
        prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;
        prog->aux->exception_boundary = func[0]->aux->exception_boundary;
        bpf_prog_jit_attempt_done(prog);
        return 0;
out_free:
        /* We failed JIT'ing, so at this point we need to unregister poke
         * descriptors from subprogs, so that kernel is not attempting to
         * patch it anymore as we're freeing the subprog JIT memory.
         */
        for (i = 0; i < prog->aux->size_poke_tab; i++) {
                map_ptr = prog->aux->poke_tab[i].tail_call.map;
                map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
        }
        /* At this point we're guaranteed that poke descriptors are not
         * live anymore. We can just unlink its descriptor table as it's
         * released with the main prog.
         */
        for (i = 0; i < env->subprog_cnt; i++) {
                if (!func[i])
                        continue;
                func[i]->aux->poke_tab = NULL;
                bpf_jit_free(func[i]);
        }
        kfree(func);
out_undo_insn:
        /* cleanup main prog to be interpreted */
        prog->jit_requested = 0;
        prog->blinding_requested = 0;
        for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
                if (!bpf_pseudo_call(insn))
                        continue;
                insn->off = 0;
                insn->imm = env->insn_aux_data[i].call_imm;
        }
        bpf_prog_jit_attempt_done(prog);
        return err;
}

static int fixup_call_args(struct bpf_verifier_env *env)
{
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
        struct bpf_prog *prog = env->prog;
        struct bpf_insn *insn = prog->insnsi;
        bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
        int i, depth;
#endif
        int err = 0;

        if (env->prog->jit_requested &&
            !bpf_prog_is_offloaded(env->prog->aux)) {
                err = jit_subprogs(env);
                if (err == 0)
                        return 0;
                if (err == -EFAULT)
                        return err;
        }
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
        if (has_kfunc_call) {
                verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
                return -EINVAL;
        }
        if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
                /* When JIT fails the progs with bpf2bpf calls and tail_calls
                 * have to be rejected, since interpreter doesn't support them yet.
                 */
                verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
                return -EINVAL;
        }
        for (i = 0; i < prog->len; i++, insn++) {
                if (bpf_pseudo_func(insn)) {
                        /* When JIT fails the progs with callback calls
                         * have to be rejected, since interpreter doesn't support them yet.
                         */
                        verbose(env, "callbacks are not allowed in non-JITed programs\n");
                        return -EINVAL;
                }

                if (!bpf_pseudo_call(insn))
                        continue;
                depth = get_callee_stack_depth(env, insn, i);
                if (depth < 0)
                        return depth;
                bpf_patch_call_args(insn, depth);
        }
        err = 0;
#endif
        return err;
}

/* replace a generic kfunc with a specialized version if necessary */
static void specialize_kfunc(struct bpf_verifier_env *env,
                             u32 func_id, u16 offset, unsigned long *addr)
{
        struct bpf_prog *prog = env->prog;
        bool seen_direct_write;
        void *xdp_kfunc;
        bool is_rdonly;

        if (bpf_dev_bound_kfunc_id(func_id)) {
                xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id);
                if (xdp_kfunc) {
                        *addr = (unsigned long)xdp_kfunc;
                        return;
                }
                /* fallback to default kfunc when not supported by netdev */
        }

        if (offset)
                return;

        if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
                seen_direct_write = env->seen_direct_write;
                is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);

                if (is_rdonly)
                        *addr = (unsigned long)bpf_dynptr_from_skb_rdonly;

                /* restore env->seen_direct_write to its original value, since
                 * may_access_direct_pkt_data mutates it
                 */
                env->seen_direct_write = seen_direct_write;
        }
}

static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
                                            u16 struct_meta_reg,
                                            u16 node_offset_reg,
                                            struct bpf_insn *insn,
                                            struct bpf_insn *insn_buf,
                                            int *cnt)
{
        struct btf_struct_meta *kptr_struct_meta = insn_aux->kptr_struct_meta;
        struct bpf_insn addr[2] = { BPF_LD_IMM64(struct_meta_reg, (long)kptr_struct_meta) };

        insn_buf[0] = addr[0];
        insn_buf[1] = addr[1];
        insn_buf[2] = BPF_MOV64_IMM(node_offset_reg, insn_aux->insert_off);
        insn_buf[3] = *insn;
        *cnt = 4;
}

static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                            struct bpf_insn *insn_buf, int insn_idx, int *cnt)
{
        const struct bpf_kfunc_desc *desc;

        if (!insn->imm) {
                verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
                return -EINVAL;
        }

        *cnt = 0;

        /* insn->imm has the btf func_id. Replace it with an offset relative to
         * __bpf_call_base, unless the JIT needs to call functions that are
         * further than 32 bits away (bpf_jit_supports_far_kfunc_call()).
         */
        desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
        if (!desc) {
                verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
                        insn->imm);
                return -EFAULT;
        }

        if (!bpf_jit_supports_far_kfunc_call())
                insn->imm = BPF_CALL_IMM(desc->addr);
        if (insn->off)
                return 0;
        if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl] ||
            desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
                struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
                struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
                u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;

                if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl] && kptr_struct_meta) {
                        verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n",
                                insn_idx);
                        return -EFAULT;
                }

                insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size);
                insn_buf[1] = addr[0];
                insn_buf[2] = addr[1];
                insn_buf[3] = *insn;
                *cnt = 4;
        } else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl] ||
                   desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] ||
                   desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
                struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
                struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };

                if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] && kptr_struct_meta) {
                        verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n",
                                insn_idx);
                        return -EFAULT;
                }

                if (desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&
                    !kptr_struct_meta) {
                        verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n",
                                insn_idx);
                        return -EFAULT;
                }

                insn_buf[0] = addr[0];
                insn_buf[1] = addr[1];
                insn_buf[2] = *insn;
                *cnt = 3;
        } else if (desc->func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
                   desc->func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
                   desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
                struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
                int struct_meta_reg = BPF_REG_3;
                int node_offset_reg = BPF_REG_4;

                /* rbtree_add has extra 'less' arg, so args-to-fixup are in diff regs */
                if (desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
                        struct_meta_reg = BPF_REG_4;
                        node_offset_reg = BPF_REG_5;
                }

                if (!kptr_struct_meta) {
                        verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n",
                                insn_idx);
                        return -EFAULT;
                }

                __fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg,
                                                node_offset_reg, insn, insn_buf, cnt);
        } else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
                   desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
                insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
                *cnt = 1;
        } else if (is_bpf_wq_set_callback_impl_kfunc(desc->func_id)) {
                struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_4, (long)env->prog->aux) };

                insn_buf[0] = ld_addrs[0];
                insn_buf[1] = ld_addrs[1];
                insn_buf[2] = *insn;
                *cnt = 3;
        }
        return 0;
}

/* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */
static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len)
{
        struct bpf_subprog_info *info = env->subprog_info;
        int cnt = env->subprog_cnt;
        struct bpf_prog *prog;

        /* We only reserve one slot for hidden subprogs in subprog_info. */
        if (env->hidden_subprog_cnt) {
                verbose(env, "verifier internal error: only one hidden subprog supported\n");
                return -EFAULT;
        }
        /* We're not patching any existing instruction, just appending the new
         * ones for the hidden subprog. Hence all of the adjustment operations
         * in bpf_patch_insn_data are no-ops.
         */
        prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len);
        if (!prog)
                return -ENOMEM;
        env->prog = prog;
        info[cnt + 1].start = info[cnt].start;
        info[cnt].start = prog->len - len + 1;
        env->subprog_cnt++;
        env->hidden_subprog_cnt++;
        return 0;
}

/* Do various post-verification rewrites in a single program pass.
 * These rewrites simplify JIT and interpreter implementations.
 */
static int do_misc_fixups(struct bpf_verifier_env *env)
{
        struct bpf_prog *prog = env->prog;
        enum bpf_attach_type eatype = prog->expected_attach_type;
        enum bpf_prog_type prog_type = resolve_prog_type(prog);
        struct bpf_insn *insn = prog->insnsi;
        const struct bpf_func_proto *fn;
        const int insn_cnt = prog->len;
        const struct bpf_map_ops *ops;
        struct bpf_insn_aux_data *aux;
        struct bpf_insn *insn_buf = env->insn_buf;
        struct bpf_prog *new_prog;
        struct bpf_map *map_ptr;
        int i, ret, cnt, delta = 0, cur_subprog = 0;
        struct bpf_subprog_info *subprogs = env->subprog_info;
        u16 stack_depth = subprogs[cur_subprog].stack_depth;
        u16 stack_depth_extra = 0;

        if (env->seen_exception && !env->exception_callback_subprog) {
                struct bpf_insn patch[] = {
                        env->prog->insnsi[insn_cnt - 1],
                        BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
                        BPF_EXIT_INSN(),
                };

                ret = add_hidden_subprog(env, patch, ARRAY_SIZE(patch));
                if (ret < 0)
                        return ret;
                prog = env->prog;
                insn = prog->insnsi;

                env->exception_callback_subprog = env->subprog_cnt - 1;
                /* Don't update insn_cnt, as add_hidden_subprog always appends insns */
                mark_subprog_exc_cb(env, env->exception_callback_subprog);
        }

        for (i = 0; i < insn_cnt;) {
                if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) {
                        if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) ||
                            (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {
                                /* convert to 32-bit mov that clears upper 32-bit */
                                insn->code = BPF_ALU | BPF_MOV | BPF_X;
                                /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */
                                insn->off = 0;
                                insn->imm = 0;
                        } /* cast from as(0) to as(1) should be handled by JIT */
                        goto next_insn;
                }

                if (env->insn_aux_data[i + delta].needs_zext)
                        /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */
                        insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code);

                /* Make sdiv/smod divide-by-minus-one exceptions impossible. */
                if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) ||
                     insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) ||
                     insn->code == (BPF_ALU | BPF_MOD | BPF_K) ||
                     insn->code == (BPF_ALU | BPF_DIV | BPF_K)) &&
                    insn->off == 1 && insn->imm == -1) {
                        bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
                        bool isdiv = BPF_OP(insn->code) == BPF_DIV;
                        struct bpf_insn *patchlet;
                        struct bpf_insn chk_and_sdiv[] = {
                                BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
                                             BPF_NEG | BPF_K, insn->dst_reg,
                                             0, 0, 0),
                        };
                        struct bpf_insn chk_and_smod[] = {
                                BPF_MOV32_IMM(insn->dst_reg, 0),
                        };

                        patchlet = isdiv ? chk_and_sdiv : chk_and_smod;
                        cnt = isdiv ? ARRAY_SIZE(chk_and_sdiv) : ARRAY_SIZE(chk_and_smod);

                        new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Make divide-by-zero and divide-by-minus-one exceptions impossible. */
                if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
                    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
                    insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
                    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
                        bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
                        bool isdiv = BPF_OP(insn->code) == BPF_DIV;
                        bool is_sdiv = isdiv && insn->off == 1;
                        bool is_smod = !isdiv && insn->off == 1;
                        struct bpf_insn *patchlet;
                        struct bpf_insn chk_and_div[] = {
                                /* [R,W]x div 0 -> 0 */
                                BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
                                             BPF_JNE | BPF_K, insn->src_reg,
                                             0, 2, 0),
                                BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
                                BPF_JMP_IMM(BPF_JA, 0, 0, 1),
                                *insn,
                        };
                        struct bpf_insn chk_and_mod[] = {
                                /* [R,W]x mod 0 -> [R,W]x */
                                BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
                                             BPF_JEQ | BPF_K, insn->src_reg,
                                             0, 1 + (is64 ? 0 : 1), 0),
                                *insn,
                                BPF_JMP_IMM(BPF_JA, 0, 0, 1),
                                BPF_MOV32_REG(insn->dst_reg, insn->dst_reg),
                        };
                        struct bpf_insn chk_and_sdiv[] = {
                                /* [R,W]x sdiv 0 -> 0
                                 * LLONG_MIN sdiv -1 -> LLONG_MIN
                                 * INT_MIN sdiv -1 -> INT_MIN
                                 */
                                BPF_MOV64_REG(BPF_REG_AX, insn->src_reg),
                                BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
                                             BPF_ADD | BPF_K, BPF_REG_AX,
                                             0, 0, 1),
                                BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
                                             BPF_JGT | BPF_K, BPF_REG_AX,
                                             0, 4, 1),
                                BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
                                             BPF_JEQ | BPF_K, BPF_REG_AX,
                                             0, 1, 0),
                                BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
                                             BPF_MOV | BPF_K, insn->dst_reg,
                                             0, 0, 0),
                                /* BPF_NEG(LLONG_MIN) == -LLONG_MIN == LLONG_MIN */
                                BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
                                             BPF_NEG | BPF_K, insn->dst_reg,
                                             0, 0, 0),
                                BPF_JMP_IMM(BPF_JA, 0, 0, 1),
                                *insn,
                        };
                        struct bpf_insn chk_and_smod[] = {
                                /* [R,W]x mod 0 -> [R,W]x */
                                /* [R,W]x mod -1 -> 0 */
                                BPF_MOV64_REG(BPF_REG_AX, insn->src_reg),
                                BPF_RAW_INSN((is64 ? BPF_ALU64 : BPF_ALU) |
                                             BPF_ADD | BPF_K, BPF_REG_AX,
                                             0, 0, 1),
                                BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
                                             BPF_JGT | BPF_K, BPF_REG_AX,
                                             0, 3, 1),
                                BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
                                             BPF_JEQ | BPF_K, BPF_REG_AX,
                                             0, 3 + (is64 ? 0 : 1), 1),
                                BPF_MOV32_IMM(insn->dst_reg, 0),
                                BPF_JMP_IMM(BPF_JA, 0, 0, 1),
                                *insn,
                                BPF_JMP_IMM(BPF_JA, 0, 0, 1),
                                BPF_MOV32_REG(insn->dst_reg, insn->dst_reg),
                        };

                        if (is_sdiv) {
                                patchlet = chk_and_sdiv;
                                cnt = ARRAY_SIZE(chk_and_sdiv);
                        } else if (is_smod) {
                                patchlet = chk_and_smod;
                                cnt = ARRAY_SIZE(chk_and_smod) - (is64 ? 2 : 0);
                        } else {
                                patchlet = isdiv ? chk_and_div : chk_and_mod;
                                cnt = isdiv ? ARRAY_SIZE(chk_and_div) :
                                              ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0);
                        }

                        new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Make it impossible to de-reference a userspace address */
                if (BPF_CLASS(insn->code) == BPF_LDX &&
                    (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
                     BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) {
                        struct bpf_insn *patch = &insn_buf[0];
                        u64 uaddress_limit = bpf_arch_uaddress_limit();

                        if (!uaddress_limit)
                                goto next_insn;

                        *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg);
                        if (insn->off)
                                *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off);
                        *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32);
                        *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2);
                        *patch++ = *insn;
                        *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
                        *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0);

                        cnt = patch - insn_buf;
                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */
                if (BPF_CLASS(insn->code) == BPF_LD &&
                    (BPF_MODE(insn->code) == BPF_ABS ||
                     BPF_MODE(insn->code) == BPF_IND)) {
                        cnt = env->ops->gen_ld_abs(insn, insn_buf);
                        if (cnt == 0 || cnt >= INSN_BUF_SIZE) {
                                verbose(env, "bpf verifier is misconfigured\n");
                                return -EINVAL;
                        }

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Rewrite pointer arithmetic to mitigate speculation attacks. */
                if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
                    insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
                        const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
                        const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
                        struct bpf_insn *patch = &insn_buf[0];
                        bool issrc, isneg, isimm;
                        u32 off_reg;

                        aux = &env->insn_aux_data[i + delta];
                        if (!aux->alu_state ||
                            aux->alu_state == BPF_ALU_NON_POINTER)
                                goto next_insn;

                        isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
                        issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
                                BPF_ALU_SANITIZE_SRC;
                        isimm = aux->alu_state & BPF_ALU_IMMEDIATE;

                        off_reg = issrc ? insn->src_reg : insn->dst_reg;
                        if (isimm) {
                                *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
                        } else {
                                if (isneg)
                                        *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
                                *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
                                *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
                                *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
                                *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
                                *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
                                *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg);
                        }
                        if (!issrc)
                                *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg);
                        insn->src_reg = BPF_REG_AX;
                        if (isneg)
                                insn->code = insn->code == code_add ?
                                             code_sub : code_add;
                        *patch++ = *insn;
                        if (issrc && isneg && !isimm)
                                *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
                        cnt = patch - insn_buf;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                if (is_may_goto_insn(insn)) {
                        int stack_off = -stack_depth - 8;

                        stack_depth_extra = 8;
                        insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off);
                        if (insn->off >= 0)
                                insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2);
                        else
                                insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1);
                        insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
                        insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off);
                        cnt = 4;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta += cnt - 1;
                        env->prog = prog = new_prog;
                        insn = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                if (insn->code != (BPF_JMP | BPF_CALL))
                        goto next_insn;
                if (insn->src_reg == BPF_PSEUDO_CALL)
                        goto next_insn;
                if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
                        ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);
                        if (ret)
                                return ret;
                        if (cnt == 0)
                                goto next_insn;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta         += cnt - 1;
                        env->prog = prog = new_prog;
                        insn          = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Skip inlining the helper call if the JIT does it. */
                if (bpf_jit_inlines_helper_call(insn->imm))
                        goto next_insn;

                if (insn->imm == BPF_FUNC_get_route_realm)
                        prog->dst_needed = 1;
                if (insn->imm == BPF_FUNC_get_prandom_u32)
                        bpf_user_rnd_init_once();
                if (insn->imm == BPF_FUNC_override_return)
                        prog->kprobe_override = 1;
                if (insn->imm == BPF_FUNC_tail_call) {
                        /* If we tail call into other programs, we
                         * cannot make any assumptions since they can
                         * be replaced dynamically during runtime in
                         * the program array.
                         */
                        prog->cb_access = 1;
                        if (!allow_tail_call_in_subprogs(env))
                                prog->aux->stack_depth = MAX_BPF_STACK;
                        prog->aux->max_pkt_offset = MAX_PACKET_OFF;

                        /* mark bpf_tail_call as different opcode to avoid
                         * conditional branch in the interpreter for every normal
                         * call and to prevent accidental JITing by JIT compiler
                         * that doesn't support bpf_tail_call yet
                         */
                        insn->imm = 0;
                        insn->code = BPF_JMP | BPF_TAIL_CALL;

                        aux = &env->insn_aux_data[i + delta];
                        if (env->bpf_capable && !prog->blinding_requested &&
                            prog->jit_requested &&
                            !bpf_map_key_poisoned(aux) &&
                            !bpf_map_ptr_poisoned(aux) &&
                            !bpf_map_ptr_unpriv(aux)) {
                                struct bpf_jit_poke_descriptor desc = {
                                        .reason = BPF_POKE_REASON_TAIL_CALL,
                                        .tail_call.map = aux->map_ptr_state.map_ptr,
                                        .tail_call.key = bpf_map_key_immediate(aux),
                                        .insn_idx = i + delta,
                                };

                                ret = bpf_jit_add_poke_descriptor(prog, &desc);
                                if (ret < 0) {
                                        verbose(env, "adding tail call poke descriptor failed\n");
                                        return ret;
                                }

                                insn->imm = ret + 1;
                                goto next_insn;
                        }

                        if (!bpf_map_ptr_unpriv(aux))
                                goto next_insn;

                        /* instead of changing every JIT dealing with tail_call
                         * emit two extra insns:
                         * if (index >= max_entries) goto out;
                         * index &= array->index_mask;
                         * to avoid out-of-bounds cpu speculation
                         */
                        if (bpf_map_ptr_poisoned(aux)) {
                                verbose(env, "tail_call abusing map_ptr\n");
                                return -EINVAL;
                        }

                        map_ptr = aux->map_ptr_state.map_ptr;
                        insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
                                                  map_ptr->max_entries, 2);
                        insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
                                                    container_of(map_ptr,
                                                                 struct bpf_array,
                                                                 map)->index_mask);
                        insn_buf[2] = *insn;
                        cnt = 3;
                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                if (insn->imm == BPF_FUNC_timer_set_callback) {
                        /* The verifier will process callback_fn as many times as necessary
                         * with different maps and the register states prepared by
                         * set_timer_callback_state will be accurate.
                         *
                         * The following use case is valid:
                         *   map1 is shared by prog1, prog2, prog3.
                         *   prog1 calls bpf_timer_init for some map1 elements
                         *   prog2 calls bpf_timer_set_callback for some map1 elements.
                         *     Those that were not bpf_timer_init-ed will return -EINVAL.
                         *   prog3 calls bpf_timer_start for some map1 elements.
                         *     Those that were not both bpf_timer_init-ed and
                         *     bpf_timer_set_callback-ed will return -EINVAL.
                         */
                        struct bpf_insn ld_addrs[2] = {
                                BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),
                        };

                        insn_buf[0] = ld_addrs[0];
                        insn_buf[1] = ld_addrs[1];
                        insn_buf[2] = *insn;
                        cnt = 3;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto patch_call_imm;
                }

                if (is_storage_get_function(insn->imm)) {
                        if (!in_sleepable(env) ||
                            env->insn_aux_data[i + delta].storage_get_func_atomic)
                                insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC);
                        else
                                insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL);
                        insn_buf[1] = *insn;
                        cnt = 2;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta += cnt - 1;
                        env->prog = prog = new_prog;
                        insn = new_prog->insnsi + i + delta;
                        goto patch_call_imm;
                }

                /* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */
                if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {
                        /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data,
                         * bpf_mem_alloc() returns a ptr to the percpu data ptr.
                         */
                        insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
                        insn_buf[1] = *insn;
                        cnt = 2;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta += cnt - 1;
                        env->prog = prog = new_prog;
                        insn = new_prog->insnsi + i + delta;
                        goto patch_call_imm;
                }

                /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
                 * and other inlining handlers are currently limited to 64 bit
                 * only.
                 */
                if (prog->jit_requested && BITS_PER_LONG == 64 &&
                    (insn->imm == BPF_FUNC_map_lookup_elem ||
                     insn->imm == BPF_FUNC_map_update_elem ||
                     insn->imm == BPF_FUNC_map_delete_elem ||
                     insn->imm == BPF_FUNC_map_push_elem   ||
                     insn->imm == BPF_FUNC_map_pop_elem    ||
                     insn->imm == BPF_FUNC_map_peek_elem   ||
                     insn->imm == BPF_FUNC_redirect_map    ||
                     insn->imm == BPF_FUNC_for_each_map_elem ||
                     insn->imm == BPF_FUNC_map_lookup_percpu_elem)) {
                        aux = &env->insn_aux_data[i + delta];
                        if (bpf_map_ptr_poisoned(aux))
                                goto patch_call_imm;

                        map_ptr = aux->map_ptr_state.map_ptr;
                        ops = map_ptr->ops;
                        if (insn->imm == BPF_FUNC_map_lookup_elem &&
                            ops->map_gen_lookup) {
                                cnt = ops->map_gen_lookup(map_ptr, insn_buf);
                                if (cnt == -EOPNOTSUPP)
                                        goto patch_map_ops_generic;
                                if (cnt <= 0 || cnt >= INSN_BUF_SIZE) {
                                        verbose(env, "bpf verifier is misconfigured\n");
                                        return -EINVAL;
                                }

                                new_prog = bpf_patch_insn_data(env, i + delta,
                                                               insn_buf, cnt);
                                if (!new_prog)
                                        return -ENOMEM;

                                delta    += cnt - 1;
                                env->prog = prog = new_prog;
                                insn      = new_prog->insnsi + i + delta;
                                goto next_insn;
                        }

                        BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
                                     (void *(*)(struct bpf_map *map, void *key))NULL));
                        BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
                                     (long (*)(struct bpf_map *map, void *key))NULL));
                        BUILD_BUG_ON(!__same_type(ops->map_update_elem,
                                     (long (*)(struct bpf_map *map, void *key, void *value,
                                              u64 flags))NULL));
                        BUILD_BUG_ON(!__same_type(ops->map_push_elem,
                                     (long (*)(struct bpf_map *map, void *value,
                                              u64 flags))NULL));
                        BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
                                     (long (*)(struct bpf_map *map, void *value))NULL));
                        BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
                                     (long (*)(struct bpf_map *map, void *value))NULL));
                        BUILD_BUG_ON(!__same_type(ops->map_redirect,
                                     (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL));
                        BUILD_BUG_ON(!__same_type(ops->map_for_each_callback,
                                     (long (*)(struct bpf_map *map,
                                              bpf_callback_t callback_fn,
                                              void *callback_ctx,
                                              u64 flags))NULL));
                        BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem,
                                     (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL));

patch_map_ops_generic:
                        switch (insn->imm) {
                        case BPF_FUNC_map_lookup_elem:
                                insn->imm = BPF_CALL_IMM(ops->map_lookup_elem);
                                goto next_insn;
                        case BPF_FUNC_map_update_elem:
                                insn->imm = BPF_CALL_IMM(ops->map_update_elem);
                                goto next_insn;
                        case BPF_FUNC_map_delete_elem:
                                insn->imm = BPF_CALL_IMM(ops->map_delete_elem);
                                goto next_insn;
                        case BPF_FUNC_map_push_elem:
                                insn->imm = BPF_CALL_IMM(ops->map_push_elem);
                                goto next_insn;
                        case BPF_FUNC_map_pop_elem:
                                insn->imm = BPF_CALL_IMM(ops->map_pop_elem);
                                goto next_insn;
                        case BPF_FUNC_map_peek_elem:
                                insn->imm = BPF_CALL_IMM(ops->map_peek_elem);
                                goto next_insn;
                        case BPF_FUNC_redirect_map:
                                insn->imm = BPF_CALL_IMM(ops->map_redirect);
                                goto next_insn;
                        case BPF_FUNC_for_each_map_elem:
                                insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);
                                goto next_insn;
                        case BPF_FUNC_map_lookup_percpu_elem:
                                insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem);
                                goto next_insn;
                        }

                        goto patch_call_imm;
                }

                /* Implement bpf_jiffies64 inline. */
                if (prog->jit_requested && BITS_PER_LONG == 64 &&
                    insn->imm == BPF_FUNC_jiffies64) {
                        struct bpf_insn ld_jiffies_addr[2] = {
                                BPF_LD_IMM64(BPF_REG_0,
                                             (unsigned long)&jiffies),
                        };

                        insn_buf[0] = ld_jiffies_addr[0];
                        insn_buf[1] = ld_jiffies_addr[1];
                        insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0,
                                                  BPF_REG_0, 0);
                        cnt = 3;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
                                                       cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
                /* Implement bpf_get_smp_processor_id() inline. */
                if (insn->imm == BPF_FUNC_get_smp_processor_id &&
                    verifier_inlines_helper_call(env, insn->imm)) {
                        /* BPF_FUNC_get_smp_processor_id inlining is an
                         * optimization, so if pcpu_hot.cpu_number is ever
                         * changed in some incompatible and hard to support
                         * way, it's fine to back out this inlining logic
                         */
#ifdef CONFIG_SMP
                        insn_buf[0] = BPF_MOV32_IMM(BPF_REG_0, (u32)(unsigned long)&pcpu_hot.cpu_number);
                        insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
                        insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0);
                        cnt = 3;
#else
                        insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
                        cnt = 1;
#endif
                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }
#endif
                /* Implement bpf_get_func_arg inline. */
                if (prog_type == BPF_PROG_TYPE_TRACING &&
                    insn->imm == BPF_FUNC_get_func_arg) {
                        /* Load nr_args from ctx - 8 */
                        insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
                        insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);
                        insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);
                        insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);
                        insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);
                        insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
                        insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);
                        insn_buf[7] = BPF_JMP_A(1);
                        insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
                        cnt = 9;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Implement bpf_get_func_ret inline. */
                if (prog_type == BPF_PROG_TYPE_TRACING &&
                    insn->imm == BPF_FUNC_get_func_ret) {
                        if (eatype == BPF_TRACE_FEXIT ||
                            eatype == BPF_MODIFY_RETURN) {
                                /* Load nr_args from ctx - 8 */
                                insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
                                insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
                                insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
                                insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
                                insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);
                                insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);
                                cnt = 6;
                        } else {
                                insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);
                                cnt = 1;
                        }

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Implement get_func_arg_cnt inline. */
                if (prog_type == BPF_PROG_TYPE_TRACING &&
                    insn->imm == BPF_FUNC_get_func_arg_cnt) {
                        /* Load nr_args from ctx - 8 */
                        insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
                        if (!new_prog)
                                return -ENOMEM;

                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Implement bpf_get_func_ip inline. */
                if (prog_type == BPF_PROG_TYPE_TRACING &&
                    insn->imm == BPF_FUNC_get_func_ip) {
                        /* Load IP address from ctx - 16 */
                        insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
                        if (!new_prog)
                                return -ENOMEM;

                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Implement bpf_get_branch_snapshot inline. */
                if (IS_ENABLED(CONFIG_PERF_EVENTS) &&
                    prog->jit_requested && BITS_PER_LONG == 64 &&
                    insn->imm == BPF_FUNC_get_branch_snapshot) {
                        /* We are dealing with the following func protos:
                         * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags);
                         * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt);
                         */
                        const u32 br_entry_size = sizeof(struct perf_branch_entry);

                        /* struct perf_branch_entry is part of UAPI and is
                         * used as an array element, so extremely unlikely to
                         * ever grow or shrink
                         */
                        BUILD_BUG_ON(br_entry_size != 24);

                        /* if (unlikely(flags)) return -EINVAL */
                        insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7);

                        /* Transform size (bytes) into number of entries (cnt = size / 24).
                         * But to avoid expensive division instruction, we implement
                         * divide-by-3 through multiplication, followed by further
                         * division by 8 through 3-bit right shift.
                         * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr.,
                         * p. 227, chapter "Unsigned Division by 3" for details and proofs.
                         *
                         * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab.
                         */
                        insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab);
                        insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0);
                        insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36);

                        /* call perf_snapshot_branch_stack implementation */
                        insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack));
                        /* if (entry_cnt == 0) return -ENOENT */
                        insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4);
                        /* return entry_cnt * sizeof(struct perf_branch_entry) */
                        insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size);
                        insn_buf[7] = BPF_JMP_A(3);
                        /* return -EINVAL; */
                        insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
                        insn_buf[9] = BPF_JMP_A(1);
                        /* return -ENOENT; */
                        insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT);
                        cnt = 11;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }

                /* Implement bpf_kptr_xchg inline */
                if (prog->jit_requested && BITS_PER_LONG == 64 &&
                    insn->imm == BPF_FUNC_kptr_xchg &&
                    bpf_jit_supports_ptr_xchg()) {
                        insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2);
                        insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
                        cnt = 2;

                        new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta    += cnt - 1;
                        env->prog = prog = new_prog;
                        insn      = new_prog->insnsi + i + delta;
                        goto next_insn;
                }
patch_call_imm:
                fn = env->ops->get_func_proto(insn->imm, env->prog);
                /* all functions that have prototype and verifier allowed
                 * programs to call them, must be real in-kernel functions
                 */
                if (!fn->func) {
                        verbose(env,
                                "kernel subsystem misconfigured func %s#%d\n",
                                func_id_name(insn->imm), insn->imm);
                        return -EFAULT;
                }
                insn->imm = fn->func - __bpf_call_base;
next_insn:
                if (subprogs[cur_subprog + 1].start == i + delta + 1) {
                        subprogs[cur_subprog].stack_depth += stack_depth_extra;
                        subprogs[cur_subprog].stack_extra = stack_depth_extra;
                        cur_subprog++;
                        stack_depth = subprogs[cur_subprog].stack_depth;
                        stack_depth_extra = 0;
                }
                i++;
                insn++;
        }

        env->prog->aux->stack_depth = subprogs[0].stack_depth;
        for (i = 0; i < env->subprog_cnt; i++) {
                int subprog_start = subprogs[i].start;
                int stack_slots = subprogs[i].stack_extra / 8;

                if (!stack_slots)
                        continue;
                if (stack_slots > 1) {
                        verbose(env, "verifier bug: stack_slots supports may_goto only\n");
                        return -EFAULT;
                }

                /* Add ST insn to subprog prologue to init extra stack */
                insn_buf[0] = BPF_ST_MEM(BPF_DW, BPF_REG_FP,
                                         -subprogs[i].stack_depth, BPF_MAX_LOOPS);
                /* Copy first actual insn to preserve it */
                insn_buf[1] = env->prog->insnsi[subprog_start];

                new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2);
                if (!new_prog)
                        return -ENOMEM;
                env->prog = prog = new_prog;
                /*
                 * If may_goto is a first insn of a prog there could be a jmp
                 * insn that points to it, hence adjust all such jmps to point
                 * to insn after BPF_ST that inits may_goto count.
                 * Adjustment will succeed because bpf_patch_insn_data() didn't fail.
                 */
                WARN_ON(adjust_jmp_off(env->prog, subprog_start, 1));
        }

        /* Since poke tab is now finalized, publish aux to tracker. */
        for (i = 0; i < prog->aux->size_poke_tab; i++) {
                map_ptr = prog->aux->poke_tab[i].tail_call.map;
                if (!map_ptr->ops->map_poke_track ||
                    !map_ptr->ops->map_poke_untrack ||
                    !map_ptr->ops->map_poke_run) {
                        verbose(env, "bpf verifier is misconfigured\n");
                        return -EINVAL;
                }

                ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
                if (ret < 0) {
                        verbose(env, "tracking tail call prog failed\n");
                        return ret;
                }
        }

        sort_kfunc_descs_by_imm_off(env->prog);

        return 0;
}

static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,
                                        int position,
                                        s32 stack_base,
                                        u32 callback_subprogno,
                                        u32 *total_cnt)
{
        s32 r6_offset = stack_base + 0 * BPF_REG_SIZE;
        s32 r7_offset = stack_base + 1 * BPF_REG_SIZE;
        s32 r8_offset = stack_base + 2 * BPF_REG_SIZE;
        int reg_loop_max = BPF_REG_6;
        int reg_loop_cnt = BPF_REG_7;
        int reg_loop_ctx = BPF_REG_8;

        struct bpf_insn *insn_buf = env->insn_buf;
        struct bpf_prog *new_prog;
        u32 callback_start;
        u32 call_insn_offset;
        s32 callback_offset;
        u32 cnt = 0;

        /* This represents an inlined version of bpf_iter.c:bpf_loop,
         * be careful to modify this code in sync.
         */

        /* Return error and jump to the end of the patch if
         * expected number of iterations is too big.
         */
        insn_buf[cnt++] = BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2);
        insn_buf[cnt++] = BPF_MOV32_IMM(BPF_REG_0, -E2BIG);
        insn_buf[cnt++] = BPF_JMP_IMM(BPF_JA, 0, 0, 16);
        /* spill R6, R7, R8 to use these as loop vars */
        insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset);
        insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset);
        insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset);
        /* initialize loop vars */
        insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_max, BPF_REG_1);
        insn_buf[cnt++] = BPF_MOV32_IMM(reg_loop_cnt, 0);
        insn_buf[cnt++] = BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3);
        /* loop header,
         * if reg_loop_cnt >= reg_loop_max skip the loop body
         */
        insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5);
        /* callback call,
         * correct callback offset would be set after patching
         */
        insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt);
        insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx);
        insn_buf[cnt++] = BPF_CALL_REL(0);
        /* increment loop counter */
        insn_buf[cnt++] = BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1);
        /* jump to loop header if callback returned 0 */
        insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6);
        /* return value of bpf_loop,
         * set R0 to the number of iterations
         */
        insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt);
        /* restore original values of R6, R7, R8 */
        insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset);
        insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset);
        insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset);

        *total_cnt = cnt;
        new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt);
        if (!new_prog)
                return new_prog;

        /* callback start is known only after patching */
        callback_start = env->subprog_info[callback_subprogno].start;
        /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */
        call_insn_offset = position + 12;
        callback_offset = callback_start - call_insn_offset - 1;
        new_prog->insnsi[call_insn_offset].imm = callback_offset;

        return new_prog;
}

static bool is_bpf_loop_call(struct bpf_insn *insn)
{
        return insn->code == (BPF_JMP | BPF_CALL) &&
                insn->src_reg == 0 &&
                insn->imm == BPF_FUNC_loop;
}

/* For all sub-programs in the program (including main) check
 * insn_aux_data to see if there are bpf_loop calls that require
 * inlining. If such calls are found the calls are replaced with a
 * sequence of instructions produced by `inline_bpf_loop` function and
 * subprog stack_depth is increased by the size of 3 registers.
 * This stack space is used to spill values of the R6, R7, R8.  These
 * registers are used to store the loop bound, counter and context
 * variables.
 */
static int optimize_bpf_loop(struct bpf_verifier_env *env)
{
        struct bpf_subprog_info *subprogs = env->subprog_info;
        int i, cur_subprog = 0, cnt, delta = 0;
        struct bpf_insn *insn = env->prog->insnsi;
        int insn_cnt = env->prog->len;
        u16 stack_depth = subprogs[cur_subprog].stack_depth;
        u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
        u16 stack_depth_extra = 0;

        for (i = 0; i < insn_cnt; i++, insn++) {
                struct bpf_loop_inline_state *inline_state =
                        &env->insn_aux_data[i + delta].loop_inline_state;

                if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) {
                        struct bpf_prog *new_prog;

                        stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup;
                        new_prog = inline_bpf_loop(env,
                                                   i + delta,
                                                   -(stack_depth + stack_depth_extra),
                                                   inline_state->callback_subprogno,
                                                   &cnt);
                        if (!new_prog)
                                return -ENOMEM;

                        delta     += cnt - 1;
                        env->prog  = new_prog;
                        insn       = new_prog->insnsi + i + delta;
                }

                if (subprogs[cur_subprog + 1].start == i + delta + 1) {
                        subprogs[cur_subprog].stack_depth += stack_depth_extra;
                        cur_subprog++;
                        stack_depth = subprogs[cur_subprog].stack_depth;
                        stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
                        stack_depth_extra = 0;
                }
        }

        env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;

        return 0;
}

/* Remove unnecessary spill/fill pairs, members of fastcall pattern,
 * adjust subprograms stack depth when possible.
 */
static int remove_fastcall_spills_fills(struct bpf_verifier_env *env)
{
        struct bpf_subprog_info *subprog = env->subprog_info;
        struct bpf_insn_aux_data *aux = env->insn_aux_data;
        struct bpf_insn *insn = env->prog->insnsi;
        int insn_cnt = env->prog->len;
        u32 spills_num;
        bool modified = false;
        int i, j;

        for (i = 0; i < insn_cnt; i++, insn++) {
                if (aux[i].fastcall_spills_num > 0) {
                        spills_num = aux[i].fastcall_spills_num;
                        /* NOPs would be removed by opt_remove_nops() */
                        for (j = 1; j <= spills_num; ++j) {
                                *(insn - j) = NOP;
                                *(insn + j) = NOP;
                        }
                        modified = true;
                }
                if ((subprog + 1)->start == i + 1) {
                        if (modified && !subprog->keep_fastcall_stack)
                                subprog->stack_depth = -subprog->fastcall_stack_off;
                        subprog++;
                        modified = false;
                }
        }

        return 0;
}

static void free_states(struct bpf_verifier_env *env)
{
        struct bpf_verifier_state_list *sl, *sln;
        int i;

        sl = env->free_list;
        while (sl) {
                sln = sl->next;
                free_verifier_state(&sl->state, false);
                kfree(sl);
                sl = sln;
        }
        env->free_list = NULL;

        if (!env->explored_states)
                return;

        for (i = 0; i < state_htab_size(env); i++) {
                sl = env->explored_states[i];

                while (sl) {
                        sln = sl->next;
                        free_verifier_state(&sl->state, false);
                        kfree(sl);
                        sl = sln;
                }
                env->explored_states[i] = NULL;
        }
}

static int do_check_common(struct bpf_verifier_env *env, int subprog)
{
        bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
        struct bpf_subprog_info *sub = subprog_info(env, subprog);
        struct bpf_verifier_state *state;
        struct bpf_reg_state *regs;
        int ret, i;

        env->prev_linfo = NULL;
        env->pass_cnt++;

        state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
        if (!state)
                return -ENOMEM;
        state->curframe = 0;
        state->speculative = false;
        state->branches = 1;
        state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
        if (!state->frame[0]) {
                kfree(state);
                return -ENOMEM;
        }
        env->cur_state = state;
        init_func_state(env, state->frame[0],
                        BPF_MAIN_FUNC /* callsite */,
                        0 /* frameno */,
                        subprog);
        state->first_insn_idx = env->subprog_info[subprog].start;
        state->last_insn_idx = -1;

        regs = state->frame[state->curframe]->regs;
        if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
                const char *sub_name = subprog_name(env, subprog);
                struct bpf_subprog_arg_info *arg;
                struct bpf_reg_state *reg;

                verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);
                ret = btf_prepare_func_args(env, subprog);
                if (ret)
                        goto out;

                if (subprog_is_exc_cb(env, subprog)) {
                        state->frame[0]->in_exception_callback_fn = true;
                        /* We have already ensured that the callback returns an integer, just
                         * like all global subprogs. We need to determine it only has a single
                         * scalar argument.
                         */
                        if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) {
                                verbose(env, "exception cb only supports single integer argument\n");
                                ret = -EINVAL;
                                goto out;
                        }
                }
                for (i = BPF_REG_1; i <= sub->arg_cnt; i++) {
                        arg = &sub->args[i - BPF_REG_1];
                        reg = &regs[i];

                        if (arg->arg_type == ARG_PTR_TO_CTX) {
                                reg->type = PTR_TO_CTX;
                                mark_reg_known_zero(env, regs, i);
                        } else if (arg->arg_type == ARG_ANYTHING) {
                                reg->type = SCALAR_VALUE;
                                mark_reg_unknown(env, regs, i);
                        } else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) {
                                /* assume unspecial LOCAL dynptr type */
                                __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen);
                        } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
                                reg->type = PTR_TO_MEM;
                                if (arg->arg_type & PTR_MAYBE_NULL)
                                        reg->type |= PTR_MAYBE_NULL;
                                mark_reg_known_zero(env, regs, i);
                                reg->mem_size = arg->mem_size;
                                reg->id = ++env->id_gen;
                        } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
                                reg->type = PTR_TO_BTF_ID;
                                if (arg->arg_type & PTR_MAYBE_NULL)
                                        reg->type |= PTR_MAYBE_NULL;
                                if (arg->arg_type & PTR_UNTRUSTED)
                                        reg->type |= PTR_UNTRUSTED;
                                if (arg->arg_type & PTR_TRUSTED)
                                        reg->type |= PTR_TRUSTED;
                                mark_reg_known_zero(env, regs, i);
                                reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */
                                reg->btf_id = arg->btf_id;
                                reg->id = ++env->id_gen;
                        } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
                                /* caller can pass either PTR_TO_ARENA or SCALAR */
                                mark_reg_unknown(env, regs, i);
                        } else {
                                WARN_ONCE(1, "BUG: unhandled arg#%d type %d\n",
                                          i - BPF_REG_1, arg->arg_type);
                                ret = -EFAULT;
                                goto out;
                        }
                }
        } else {
                /* if main BPF program has associated BTF info, validate that
                 * it's matching expected signature, and otherwise mark BTF
                 * info for main program as unreliable
                 */
                if (env->prog->aux->func_info_aux) {
                        ret = btf_prepare_func_args(env, 0);
                        if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX)
                                env->prog->aux->func_info_aux[0].unreliable = true;
                }

                /* 1st arg to a function */
                regs[BPF_REG_1].type = PTR_TO_CTX;
                mark_reg_known_zero(env, regs, BPF_REG_1);
        }

        ret = do_check(env);
out:
        /* check for NULL is necessary, since cur_state can be freed inside
         * do_check() under memory pressure.
         */
        if (env->cur_state) {
                free_verifier_state(env->cur_state, true);
                env->cur_state = NULL;
        }
        while (!pop_stack(env, NULL, NULL, false));
        if (!ret && pop_log)
                bpf_vlog_reset(&env->log, 0);
        free_states(env);
        return ret;
}

/* Lazily verify all global functions based on their BTF, if they are called
 * from main BPF program or any of subprograms transitively.
 * BPF global subprogs called from dead code are not validated.
 * All callable global functions must pass verification.
 * Otherwise the whole program is rejected.
 * Consider:
 * int bar(int);
 * int foo(int f)
 * {
 *    return bar(f);
 * }
 * int bar(int b)
 * {
 *    ...
 * }
 * foo() will be verified first for R1=any_scalar_value. During verification it
 * will be assumed that bar() already verified successfully and call to bar()
 * from foo() will be checked for type match only. Later bar() will be verified
 * independently to check that it's safe for R1=any_scalar_value.
 */
static int do_check_subprogs(struct bpf_verifier_env *env)
{
        struct bpf_prog_aux *aux = env->prog->aux;
        struct bpf_func_info_aux *sub_aux;
        int i, ret, new_cnt;

        if (!aux->func_info)
                return 0;

        /* exception callback is presumed to be always called */
        if (env->exception_callback_subprog)
                subprog_aux(env, env->exception_callback_subprog)->called = true;

again:
        new_cnt = 0;
        for (i = 1; i < env->subprog_cnt; i++) {
                if (!subprog_is_global(env, i))
                        continue;

                sub_aux = subprog_aux(env, i);
                if (!sub_aux->called || sub_aux->verified)
                        continue;

                env->insn_idx = env->subprog_info[i].start;
                WARN_ON_ONCE(env->insn_idx == 0);
                ret = do_check_common(env, i);
                if (ret) {
                        return ret;
                } else if (env->log.level & BPF_LOG_LEVEL) {
                        verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n",
                                i, subprog_name(env, i));
                }

                /* We verified new global subprog, it might have called some
                 * more global subprogs that we haven't verified yet, so we
                 * need to do another pass over subprogs to verify those.
                 */
                sub_aux->verified = true;
                new_cnt++;
        }

        /* We can't loop forever as we verify at least one global subprog on
         * each pass.
         */
        if (new_cnt)
                goto again;

        return 0;
}

static int do_check_main(struct bpf_verifier_env *env)
{
        int ret;

        env->insn_idx = 0;
        ret = do_check_common(env, 0);
        if (!ret)
                env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
        return ret;
}


static void print_verification_stats(struct bpf_verifier_env *env)
{
        int i;

        if (env->log.level & BPF_LOG_STATS) {
                verbose(env, "verification time %lld usec\n",
                        div_u64(env->verification_time, 1000));
                verbose(env, "stack depth ");
                for (i = 0; i < env->subprog_cnt; i++) {
                        u32 depth = env->subprog_info[i].stack_depth;

                        verbose(env, "%d", depth);
                        if (i + 1 < env->subprog_cnt)
                                verbose(env, "+");
                }
                verbose(env, "\n");
        }
        verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
                "total_states %d peak_states %d mark_read %d\n",
                env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
                env->max_states_per_insn, env->total_states,
                env->peak_states, env->longest_mark_read_walk);
}

static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
{
        const struct btf_type *t, *func_proto;
        const struct bpf_struct_ops_desc *st_ops_desc;
        const struct bpf_struct_ops *st_ops;
        const struct btf_member *member;
        struct bpf_prog *prog = env->prog;
        u32 btf_id, member_idx;
        struct btf *btf;
        const char *mname;
        int err;

        if (!prog->gpl_compatible) {
                verbose(env, "struct ops programs must have a GPL compatible license\n");
                return -EINVAL;
        }

        if (!prog->aux->attach_btf_id)
                return -ENOTSUPP;

        btf = prog->aux->attach_btf;
        if (btf_is_module(btf)) {
                /* Make sure st_ops is valid through the lifetime of env */
                env->attach_btf_mod = btf_try_get_module(btf);
                if (!env->attach_btf_mod) {
                        verbose(env, "struct_ops module %s is not found\n",
                                btf_get_name(btf));
                        return -ENOTSUPP;
                }
        }

        btf_id = prog->aux->attach_btf_id;
        st_ops_desc = bpf_struct_ops_find(btf, btf_id);
        if (!st_ops_desc) {
                verbose(env, "attach_btf_id %u is not a supported struct\n",
                        btf_id);
                return -ENOTSUPP;
        }
        st_ops = st_ops_desc->st_ops;

        t = st_ops_desc->type;
        member_idx = prog->expected_attach_type;
        if (member_idx >= btf_type_vlen(t)) {
                verbose(env, "attach to invalid member idx %u of struct %s\n",
                        member_idx, st_ops->name);
                return -EINVAL;
        }

        member = &btf_type_member(t)[member_idx];
        mname = btf_name_by_offset(btf, member->name_off);
        func_proto = btf_type_resolve_func_ptr(btf, member->type,
                                               NULL);
        if (!func_proto) {
                verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
                        mname, member_idx, st_ops->name);
                return -EINVAL;
        }

        err = bpf_struct_ops_supported(st_ops, __btf_member_bit_offset(t, member) / 8);
        if (err) {
                verbose(env, "attach to unsupported member %s of struct %s\n",
                        mname, st_ops->name);
                return err;
        }

        if (st_ops->check_member) {
                err = st_ops->check_member(t, member, prog);

                if (err) {
                        verbose(env, "attach to unsupported member %s of struct %s\n",
                                mname, st_ops->name);
                        return err;
                }
        }

        if (prog->aux->priv_stack_requested && !bpf_jit_supports_private_stack()) {
                verbose(env, "Private stack not supported by jit\n");
                return -EACCES;
        }

        /* btf_ctx_access() used this to provide argument type info */
        prog->aux->ctx_arg_info =
                st_ops_desc->arg_info[member_idx].info;
        prog->aux->ctx_arg_info_size =
                st_ops_desc->arg_info[member_idx].cnt;

        prog->aux->attach_func_proto = func_proto;
        prog->aux->attach_func_name = mname;
        env->ops = st_ops->verifier_ops;

        return 0;
}
#define SECURITY_PREFIX "security_"

static int check_attach_modify_return(unsigned long addr, const char *func_name)
{
        if (within_error_injection_list(addr) ||
            !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
                return 0;

        return -EINVAL;
}

/* list of non-sleepable functions that are otherwise on
 * ALLOW_ERROR_INJECTION list
 */
BTF_SET_START(btf_non_sleepable_error_inject)
/* Three functions below can be called from sleepable and non-sleepable context.
 * Assume non-sleepable from bpf safety point of view.
 */
BTF_ID(func, __filemap_add_folio)
#ifdef CONFIG_FAIL_PAGE_ALLOC
BTF_ID(func, should_fail_alloc_page)
#endif
#ifdef CONFIG_FAILSLAB
BTF_ID(func, should_failslab)
#endif
BTF_SET_END(btf_non_sleepable_error_inject)

static int check_non_sleepable_error_inject(u32 btf_id)
{
        return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
}

int bpf_check_attach_target(struct bpf_verifier_log *log,
                            const struct bpf_prog *prog,
                            const struct bpf_prog *tgt_prog,
                            u32 btf_id,
                            struct bpf_attach_target_info *tgt_info)
{
        bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
        bool prog_tracing = prog->type == BPF_PROG_TYPE_TRACING;
        char trace_symbol[KSYM_SYMBOL_LEN];
        const char prefix[] = "btf_trace_";
        struct bpf_raw_event_map *btp;
        int ret = 0, subprog = -1, i;
        const struct btf_type *t;
        bool conservative = true;
        const char *tname, *fname;
        struct btf *btf;
        long addr = 0;
        struct module *mod = NULL;

        if (!btf_id) {
                bpf_log(log, "Tracing programs must provide btf_id\n");
                return -EINVAL;
        }
        btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf;
        if (!btf) {
                bpf_log(log,
                        "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");
                return -EINVAL;
        }
        t = btf_type_by_id(btf, btf_id);
        if (!t) {
                bpf_log(log, "attach_btf_id %u is invalid\n", btf_id);
                return -EINVAL;
        }
        tname = btf_name_by_offset(btf, t->name_off);
        if (!tname) {
                bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id);
                return -EINVAL;
        }
        if (tgt_prog) {
                struct bpf_prog_aux *aux = tgt_prog->aux;
                bool tgt_changes_pkt_data;

                if (bpf_prog_is_dev_bound(prog->aux) &&
                    !bpf_prog_dev_bound_match(prog, tgt_prog)) {
                        bpf_log(log, "Target program bound device mismatch");
                        return -EINVAL;
                }

                for (i = 0; i < aux->func_info_cnt; i++)
                        if (aux->func_info[i].type_id == btf_id) {
                                subprog = i;
                                break;
                        }
                if (subprog == -1) {
                        bpf_log(log, "Subprog %s doesn't exist\n", tname);
                        return -EINVAL;
                }
                if (aux->func && aux->func[subprog]->aux->exception_cb) {
                        bpf_log(log,
                                "%s programs cannot attach to exception callback\n",
                                prog_extension ? "Extension" : "FENTRY/FEXIT");
                        return -EINVAL;
                }
                conservative = aux->func_info_aux[subprog].unreliable;
                if (prog_extension) {
                        if (conservative) {
                                bpf_log(log,
                                        "Cannot replace static functions\n");
                                return -EINVAL;
                        }
                        if (!prog->jit_requested) {
                                bpf_log(log,
                                        "Extension programs should be JITed\n");
                                return -EINVAL;
                        }
                        tgt_changes_pkt_data = aux->func
                                               ? aux->func[subprog]->aux->changes_pkt_data
                                               : aux->changes_pkt_data;
                        if (prog->aux->changes_pkt_data && !tgt_changes_pkt_data) {
                                bpf_log(log,
                                        "Extension program changes packet data, while original does not\n");
                                return -EINVAL;
                        }
                }
                if (!tgt_prog->jited) {
                        bpf_log(log, "Can attach to only JITed progs\n");
                        return -EINVAL;
                }
                if (prog_tracing) {
                        if (aux->attach_tracing_prog) {
                                /*
                                 * Target program is an fentry/fexit which is already attached
                                 * to another tracing program. More levels of nesting
                                 * attachment are not allowed.
                                 */
                                bpf_log(log, "Cannot nest tracing program attach more than once\n");
                                return -EINVAL;
                        }
                } else if (tgt_prog->type == prog->type) {
                        /*
                         * To avoid potential call chain cycles, prevent attaching of a
                         * program extension to another extension. It's ok to attach
                         * fentry/fexit to extension program.
                         */
                        bpf_log(log, "Cannot recursively attach\n");
                        return -EINVAL;
                }
                if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
                    prog_extension &&
                    (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
                     tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) {
                        /* Program extensions can extend all program types
                         * except fentry/fexit. The reason is the following.
                         * The fentry/fexit programs are used for performance
                         * analysis, stats and can be attached to any program
                         * type. When extension program is replacing XDP function
                         * it is necessary to allow performance analysis of all
                         * functions. Both original XDP program and its program
                         * extension. Hence attaching fentry/fexit to
                         * BPF_PROG_TYPE_EXT is allowed. If extending of
                         * fentry/fexit was allowed it would be possible to create
                         * long call chain fentry->extension->fentry->extension
                         * beyond reasonable stack size. Hence extending fentry
                         * is not allowed.
                         */
                        bpf_log(log, "Cannot extend fentry/fexit\n");
                        return -EINVAL;
                }
        } else {
                if (prog_extension) {
                        bpf_log(log, "Cannot replace kernel functions\n");
                        return -EINVAL;
                }
        }

        switch (prog->expected_attach_type) {
        case BPF_TRACE_RAW_TP:
                if (tgt_prog) {
                        bpf_log(log,
                                "Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
                        return -EINVAL;
                }
                if (!btf_type_is_typedef(t)) {
                        bpf_log(log, "attach_btf_id %u is not a typedef\n",
                                btf_id);
                        return -EINVAL;
                }
                if (strncmp(prefix, tname, sizeof(prefix) - 1)) {
                        bpf_log(log, "attach_btf_id %u points to wrong type name %s\n",
                                btf_id, tname);
                        return -EINVAL;
                }
                tname += sizeof(prefix) - 1;

                /* The func_proto of "btf_trace_##tname" is generated from typedef without argument
                 * names. Thus using bpf_raw_event_map to get argument names.
                 */
                btp = bpf_get_raw_tracepoint(tname);
                if (!btp)
                        return -EINVAL;
                fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL,
                                        trace_symbol);
                bpf_put_raw_tracepoint(btp);

                if (fname)
                        ret = btf_find_by_name_kind(btf, fname, BTF_KIND_FUNC);

                if (!fname || ret < 0) {
                        bpf_log(log, "Cannot find btf of tracepoint template, fall back to %s%s.\n",
                                prefix, tname);
                        t = btf_type_by_id(btf, t->type);
                        if (!btf_type_is_ptr(t))
                                /* should never happen in valid vmlinux build */
                                return -EINVAL;
                } else {
                        t = btf_type_by_id(btf, ret);
                        if (!btf_type_is_func(t))
                                /* should never happen in valid vmlinux build */
                                return -EINVAL;
                }

                t = btf_type_by_id(btf, t->type);
                if (!btf_type_is_func_proto(t))
                        /* should never happen in valid vmlinux build */
                        return -EINVAL;

                break;
        case BPF_TRACE_ITER:
                if (!btf_type_is_func(t)) {
                        bpf_log(log, "attach_btf_id %u is not a function\n",
                                btf_id);
                        return -EINVAL;
                }
                t = btf_type_by_id(btf, t->type);
                if (!btf_type_is_func_proto(t))
                        return -EINVAL;
                ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
                if (ret)
                        return ret;
                break;
        default:
                if (!prog_extension)
                        return -EINVAL;
                fallthrough;
        case BPF_MODIFY_RETURN:
        case BPF_LSM_MAC:
        case BPF_LSM_CGROUP:
        case BPF_TRACE_FENTRY:
        case BPF_TRACE_FEXIT:
                if (!btf_type_is_func(t)) {
                        bpf_log(log, "attach_btf_id %u is not a function\n",
                                btf_id);
                        return -EINVAL;
                }
                if (prog_extension &&
                    btf_check_type_match(log, prog, btf, t))
                        return -EINVAL;
                t = btf_type_by_id(btf, t->type);
                if (!btf_type_is_func_proto(t))
                        return -EINVAL;

                if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
                    (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
                     prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
                        return -EINVAL;

                if (tgt_prog && conservative)
                        t = NULL;

                ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
                if (ret < 0)
                        return ret;

                if (tgt_prog) {
                        if (subprog == 0)
                                addr = (long) tgt_prog->bpf_func;
                        else
                                addr = (long) tgt_prog->aux->func[subprog]->bpf_func;
                } else {
                        if (btf_is_module(btf)) {
                                mod = btf_try_get_module(btf);
                                if (mod)
                                        addr = find_kallsyms_symbol_value(mod, tname);
                                else
                                        addr = 0;
                        } else {
                                addr = kallsyms_lookup_name(tname);
                        }
                        if (!addr) {
                                module_put(mod);
                                bpf_log(log,
                                        "The address of function %s cannot be found\n",
                                        tname);
                                return -ENOENT;
                        }
                }

                if (prog->sleepable) {
                        ret = -EINVAL;
                        switch (prog->type) {
                        case BPF_PROG_TYPE_TRACING:

                                /* fentry/fexit/fmod_ret progs can be sleepable if they are
                                 * attached to ALLOW_ERROR_INJECTION and are not in denylist.
                                 */
                                if (!check_non_sleepable_error_inject(btf_id) &&
                                    within_error_injection_list(addr))
                                        ret = 0;
                                /* fentry/fexit/fmod_ret progs can also be sleepable if they are
                                 * in the fmodret id set with the KF_SLEEPABLE flag.
                                 */
                                else {
                                        u32 *flags = btf_kfunc_is_modify_return(btf, btf_id,
                                                                                prog);

                                        if (flags && (*flags & KF_SLEEPABLE))
                                                ret = 0;
                                }
                                break;
                        case BPF_PROG_TYPE_LSM:
                                /* LSM progs check that they are attached to bpf_lsm_*() funcs.
                                 * Only some of them are sleepable.
                                 */
                                if (bpf_lsm_is_sleepable_hook(btf_id))
                                        ret = 0;
                                break;
                        default:
                                break;
                        }
                        if (ret) {
                                module_put(mod);
                                bpf_log(log, "%s is not sleepable\n", tname);
                                return ret;
                        }
                } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
                        if (tgt_prog) {
                                module_put(mod);
                                bpf_log(log, "can't modify return codes of BPF programs\n");
                                return -EINVAL;
                        }
                        ret = -EINVAL;
                        if (btf_kfunc_is_modify_return(btf, btf_id, prog) ||
                            !check_attach_modify_return(addr, tname))
                                ret = 0;
                        if (ret) {
                                module_put(mod);
                                bpf_log(log, "%s() is not modifiable\n", tname);
                                return ret;
                        }
                }

                break;
        }
        tgt_info->tgt_addr = addr;
        tgt_info->tgt_name = tname;
        tgt_info->tgt_type = t;
        tgt_info->tgt_mod = mod;
        return 0;
}

BTF_SET_START(btf_id_deny)
BTF_ID_UNUSED
#ifdef CONFIG_SMP
BTF_ID(func, migrate_disable)
BTF_ID(func, migrate_enable)
#endif
#if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU
BTF_ID(func, rcu_read_unlock_strict)
#endif
#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)
BTF_ID(func, preempt_count_add)
BTF_ID(func, preempt_count_sub)
#endif
#ifdef CONFIG_PREEMPT_RCU
BTF_ID(func, __rcu_read_lock)
BTF_ID(func, __rcu_read_unlock)
#endif
BTF_SET_END(btf_id_deny)

static bool can_be_sleepable(struct bpf_prog *prog)
{
        if (prog->type == BPF_PROG_TYPE_TRACING) {
                switch (prog->expected_attach_type) {
                case BPF_TRACE_FENTRY:
                case BPF_TRACE_FEXIT:
                case BPF_MODIFY_RETURN:
                case BPF_TRACE_ITER:
                        return true;
                default:
                        return false;
                }
        }
        return prog->type == BPF_PROG_TYPE_LSM ||
               prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ ||
               prog->type == BPF_PROG_TYPE_STRUCT_OPS;
}

static int check_attach_btf_id(struct bpf_verifier_env *env)
{
        struct bpf_prog *prog = env->prog;
        struct bpf_prog *tgt_prog = prog->aux->dst_prog;
        struct bpf_attach_target_info tgt_info = {};
        u32 btf_id = prog->aux->attach_btf_id;
        struct bpf_trampoline *tr;
        int ret;
        u64 key;

        if (prog->type == BPF_PROG_TYPE_SYSCALL) {
                if (prog->sleepable)
                        /* attach_btf_id checked to be zero already */
                        return 0;
                verbose(env, "Syscall programs can only be sleepable\n");
                return -EINVAL;
        }

        if (prog->sleepable && !can_be_sleepable(prog)) {
                verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n");
                return -EINVAL;
        }

        if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
                return check_struct_ops_btf_id(env);

        if (prog->type != BPF_PROG_TYPE_TRACING &&
            prog->type != BPF_PROG_TYPE_LSM &&
            prog->type != BPF_PROG_TYPE_EXT)
                return 0;

        ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
        if (ret)
                return ret;

        if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
                /* to make freplace equivalent to their targets, they need to
                 * inherit env->ops and expected_attach_type for the rest of the
                 * verification
                 */
                env->ops = bpf_verifier_ops[tgt_prog->type];
                prog->expected_attach_type = tgt_prog->expected_attach_type;
        }

        /* store info about the attachment target that will be used later */
        prog->aux->attach_func_proto = tgt_info.tgt_type;
        prog->aux->attach_func_name = tgt_info.tgt_name;
        prog->aux->mod = tgt_info.tgt_mod;

        if (tgt_prog) {
                prog->aux->saved_dst_prog_type = tgt_prog->type;
                prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
        }

        if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
                prog->aux->attach_btf_trace = true;
                return 0;
        } else if (prog->expected_attach_type == BPF_TRACE_ITER) {
                if (!bpf_iter_prog_supported(prog))
                        return -EINVAL;
                return 0;
        }

        if (prog->type == BPF_PROG_TYPE_LSM) {
                ret = bpf_lsm_verify_prog(&env->log, prog);
                if (ret < 0)
                        return ret;
        } else if (prog->type == BPF_PROG_TYPE_TRACING &&
                   btf_id_set_contains(&btf_id_deny, btf_id)) {
                return -EINVAL;
        }

        key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
        tr = bpf_trampoline_get(key, &tgt_info);
        if (!tr)
                return -ENOMEM;

        if (tgt_prog && tgt_prog->aux->tail_call_reachable)
                tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX;

        prog->aux->dst_trampoline = tr;
        return 0;
}

struct btf *bpf_get_btf_vmlinux(void)
{
        if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
                mutex_lock(&bpf_verifier_lock);
                if (!btf_vmlinux)
                        btf_vmlinux = btf_parse_vmlinux();
                mutex_unlock(&bpf_verifier_lock);
        }
        return btf_vmlinux;
}

/*
 * The add_fd_from_fd_array() is executed only if fd_array_cnt is non-zero. In
 * this case expect that every file descriptor in the array is either a map or
 * a BTF. Everything else is considered to be trash.
 */
static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd)
{
        struct bpf_map *map;
        struct btf *btf;
        CLASS(fd, f)(fd);
        int err;

        map = __bpf_map_get(f);
        if (!IS_ERR(map)) {
                err = __add_used_map(env, map);
                if (err < 0)
                        return err;
                return 0;
        }

        btf = __btf_get_by_fd(f);
        if (!IS_ERR(btf)) {
                err = __add_used_btf(env, btf);
                if (err < 0)
                        return err;
                return 0;
        }

        verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd);
        return PTR_ERR(map);
}

static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr)
{
        size_t size = sizeof(int);
        int ret;
        int fd;
        u32 i;

        env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);

        /*
         * The only difference between old (no fd_array_cnt is given) and new
         * APIs is that in the latter case the fd_array is expected to be
         * continuous and is scanned for map fds right away
         */
        if (!attr->fd_array_cnt)
                return 0;

        /* Check for integer overflow */
        if (attr->fd_array_cnt >= (U32_MAX / size)) {
                verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt);
                return -EINVAL;
        }

        for (i = 0; i < attr->fd_array_cnt; i++) {
                if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size))
                        return -EFAULT;

                ret = add_fd_from_fd_array(env, fd);
                if (ret)
                        return ret;
        }

        return 0;
}

int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
{
        u64 start_time = ktime_get_ns();
        struct bpf_verifier_env *env;
        int i, len, ret = -EINVAL, err;
        u32 log_true_size;
        bool is_priv;

        /* no program is valid */
        if (ARRAY_SIZE(bpf_verifier_ops) == 0)
                return -EINVAL;

        /* 'struct bpf_verifier_env' can be global, but since it's not small,
         * allocate/free it every time bpf_check() is called
         */
        env = kvzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
        if (!env)
                return -ENOMEM;

        env->bt.env = env;

        len = (*prog)->len;
        env->insn_aux_data =
                vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
        ret = -ENOMEM;
        if (!env->insn_aux_data)
                goto err_free_env;
        for (i = 0; i < len; i++)
                env->insn_aux_data[i].orig_idx = i;
        env->prog = *prog;
        env->ops = bpf_verifier_ops[env->prog->type];

        env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token);
        env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token);
        env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);
        env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);
        env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);

        bpf_get_btf_vmlinux();

        /* grab the mutex to protect few globals used by verifier */
        if (!is_priv)
                mutex_lock(&bpf_verifier_lock);

        /* user could have requested verbose verifier output
         * and supplied buffer to store the verification trace
         */
        ret = bpf_vlog_init(&env->log, attr->log_level,
                            (char __user *) (unsigned long) attr->log_buf,
                            attr->log_size);
        if (ret)
                goto err_unlock;

        ret = process_fd_array(env, attr, uattr);
        if (ret)
                goto skip_full_check;

        mark_verifier_state_clean(env);

        if (IS_ERR(btf_vmlinux)) {
                /* Either gcc or pahole or kernel are broken. */
                verbose(env, "in-kernel BTF is malformed\n");
                ret = PTR_ERR(btf_vmlinux);
                goto skip_full_check;
        }

        env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
        if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
                env->strict_alignment = true;
        if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
                env->strict_alignment = false;

        if (is_priv)
                env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
        env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;

        env->explored_states = kvcalloc(state_htab_size(env),
                                       sizeof(struct bpf_verifier_state_list *),
                                       GFP_USER);
        ret = -ENOMEM;
        if (!env->explored_states)
                goto skip_full_check;

        ret = check_btf_info_early(env, attr, uattr);
        if (ret < 0)
                goto skip_full_check;

        ret = add_subprog_and_kfunc(env);
        if (ret < 0)
                goto skip_full_check;

        ret = check_subprogs(env);
        if (ret < 0)
                goto skip_full_check;

        ret = check_btf_info(env, attr, uattr);
        if (ret < 0)
                goto skip_full_check;

        ret = resolve_pseudo_ldimm64(env);
        if (ret < 0)
                goto skip_full_check;

        if (bpf_prog_is_offloaded(env->prog->aux)) {
                ret = bpf_prog_offload_verifier_prep(env->prog);
                if (ret)
                        goto skip_full_check;
        }

        ret = check_cfg(env);
        if (ret < 0)
                goto skip_full_check;

        ret = check_attach_btf_id(env);
        if (ret)
                goto skip_full_check;

        ret = mark_fastcall_patterns(env);
        if (ret < 0)
                goto skip_full_check;

        ret = do_check_main(env);
        ret = ret ?: do_check_subprogs(env);

        if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux))
                ret = bpf_prog_offload_finalize(env);

skip_full_check:
        kvfree(env->explored_states);

        /* might decrease stack depth, keep it before passes that
         * allocate additional slots.
         */
        if (ret == 0)
                ret = remove_fastcall_spills_fills(env);

        if (ret == 0)
                ret = check_max_stack_depth(env);

        /* instruction rewrites happen after this point */
        if (ret == 0)
                ret = optimize_bpf_loop(env);

        if (is_priv) {
                if (ret == 0)
                        opt_hard_wire_dead_code_branches(env);
                if (ret == 0)
                        ret = opt_remove_dead_code(env);
                if (ret == 0)
                        ret = opt_remove_nops(env);
        } else {
                if (ret == 0)
                        sanitize_dead_code(env);
        }

        if (ret == 0)
                /* program is valid, convert *(u32*)(ctx + off) accesses */
                ret = convert_ctx_accesses(env);

        if (ret == 0)
                ret = do_misc_fixups(env);

        /* do 32-bit optimization after insn patching has done so those patched
         * insns could be handled correctly.
         */
        if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) {
                ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);
                env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret
                                                                     : false;
        }

        if (ret == 0)
                ret = fixup_call_args(env);

        env->verification_time = ktime_get_ns() - start_time;
        print_verification_stats(env);
        env->prog->aux->verified_insns = env->insn_processed;

        /* preserve original error even if log finalization is successful */
        err = bpf_vlog_finalize(&env->log, &log_true_size);
        if (err)
                ret = err;

        if (uattr_size >= offsetofend(union bpf_attr, log_true_size) &&
            copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, log_true_size),
                                  &log_true_size, sizeof(log_true_size))) {
                ret = -EFAULT;
                goto err_release_maps;
        }

        if (ret)
                goto err_release_maps;

        if (env->used_map_cnt) {
                /* if program passed verifier, update used_maps in bpf_prog_info */
                env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
                                                          sizeof(env->used_maps[0]),
                                                          GFP_KERNEL);

                if (!env->prog->aux->used_maps) {
                        ret = -ENOMEM;
                        goto err_release_maps;
                }

                memcpy(env->prog->aux->used_maps, env->used_maps,
                       sizeof(env->used_maps[0]) * env->used_map_cnt);
                env->prog->aux->used_map_cnt = env->used_map_cnt;
        }
        if (env->used_btf_cnt) {
                /* if program passed verifier, update used_btfs in bpf_prog_aux */
                env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,
                                                          sizeof(env->used_btfs[0]),
                                                          GFP_KERNEL);
                if (!env->prog->aux->used_btfs) {
                        ret = -ENOMEM;
                        goto err_release_maps;
                }

                memcpy(env->prog->aux->used_btfs, env->used_btfs,
                       sizeof(env->used_btfs[0]) * env->used_btf_cnt);
                env->prog->aux->used_btf_cnt = env->used_btf_cnt;
        }
        if (env->used_map_cnt || env->used_btf_cnt) {
                /* program is valid. Convert pseudo bpf_ld_imm64 into generic
                 * bpf_ld_imm64 instructions
                 */
                convert_pseudo_ld_imm64(env);
        }

        adjust_btf_func(env);

err_release_maps:
        if (!env->prog->aux->used_maps)
                /* if we didn't copy map pointers into bpf_prog_info, release
                 * them now. Otherwise free_used_maps() will release them.
                 */
                release_maps(env);
        if (!env->prog->aux->used_btfs)
                release_btfs(env);

        /* extension progs temporarily inherit the attach_type of their targets
           for verification purposes, so set it back to zero before returning
         */
        if (env->prog->type == BPF_PROG_TYPE_EXT)
                env->prog->expected_attach_type = 0;

        *prog = env->prog;

        module_put(env->attach_btf_mod);
err_unlock:
        if (!is_priv)
                mutex_unlock(&bpf_verifier_lock);
        vfree(env->insn_aux_data);
        kvfree(env->insn_hist);
err_free_env:
        kvfree(env);
        return ret;
}
































   13 

   13 

   13 

   13 









   13 


   13 












   13 
    6 





   13 

   13 




   13 




















    9 



    9 









    9 




   21 
   21 








    1 


    1 




    1 









    1 
    1 

    1 










































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
// SPDX-License-Identifier: GPL-2.0
/*
 * KVM coalesced MMIO
 *
 * Copyright (c) 2008 Bull S.A.S.
 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
 *
 *  Author: Laurent Vivier <Laurent.Vivier@bull.net>
 *
 */

#include <kvm/iodev.h>

#include <linux/kvm_host.h>
#include <linux/slab.h>
#include <linux/kvm.h>

#include "coalesced_mmio.h"

static inline struct kvm_coalesced_mmio_dev *to_mmio(struct kvm_io_device *dev)
{
        return container_of(dev, struct kvm_coalesced_mmio_dev, dev);
}

static int coalesced_mmio_in_range(struct kvm_coalesced_mmio_dev *dev,
                                   gpa_t addr, int len)
{
        /* is it in a batchable area ?
         * (addr,len) is fully included in
         * (zone->addr, zone->size)
         */
        if (len < 0)
                return 0;
        if (addr + len < addr)
                return 0;
        if (addr < dev->zone.addr)
                return 0;
        if (addr + len > dev->zone.addr + dev->zone.size)
                return 0;
        return 1;
}

static int coalesced_mmio_write(struct kvm_vcpu *vcpu,
                                struct kvm_io_device *this, gpa_t addr,
                                int len, const void *val)
{
        struct kvm_coalesced_mmio_dev *dev = to_mmio(this);
        struct kvm_coalesced_mmio_ring *ring = dev->kvm->coalesced_mmio_ring;
        __u32 insert;

        if (!coalesced_mmio_in_range(dev, addr, len))
                return -EOPNOTSUPP;

        spin_lock(&dev->kvm->ring_lock);

        /*
         * last is the index of the entry to fill.  Verify userspace hasn't
         * set last to be out of range, and that there is room in the ring.
         * Leave one entry free in the ring so that userspace can differentiate
         * between an empty ring and a full ring.
         */
        insert = READ_ONCE(ring->last);
        if (insert >= KVM_COALESCED_MMIO_MAX ||
            (insert + 1) % KVM_COALESCED_MMIO_MAX == READ_ONCE(ring->first)) {
                spin_unlock(&dev->kvm->ring_lock);
                return -EOPNOTSUPP;
        }

        /* copy data in first free entry of the ring */

        ring->coalesced_mmio[insert].phys_addr = addr;
        ring->coalesced_mmio[insert].len = len;
        memcpy(ring->coalesced_mmio[insert].data, val, len);
        ring->coalesced_mmio[insert].pio = dev->zone.pio;
        smp_wmb();
        ring->last = (insert + 1) % KVM_COALESCED_MMIO_MAX;
        spin_unlock(&dev->kvm->ring_lock);
        return 0;
}

static void coalesced_mmio_destructor(struct kvm_io_device *this)
{
        struct kvm_coalesced_mmio_dev *dev = to_mmio(this);

        list_del(&dev->list);

        kfree(dev);
}

static const struct kvm_io_device_ops coalesced_mmio_ops = {
        .write      = coalesced_mmio_write,
        .destructor = coalesced_mmio_destructor,
};

int kvm_coalesced_mmio_init(struct kvm *kvm)
{
        struct page *page;

        page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
        if (!page)
                return -ENOMEM;

        kvm->coalesced_mmio_ring = page_address(page);

        /*
         * We're using this spinlock to sync access to the coalesced ring.
         * The list doesn't need its own lock since device registration and
         * unregistration should only happen when kvm->slots_lock is held.
         */
        spin_lock_init(&kvm->ring_lock);
        INIT_LIST_HEAD(&kvm->coalesced_zones);

        return 0;
}

void kvm_coalesced_mmio_free(struct kvm *kvm)
{
        if (kvm->coalesced_mmio_ring)
                free_page((unsigned long)kvm->coalesced_mmio_ring);
}

int kvm_vm_ioctl_register_coalesced_mmio(struct kvm *kvm,
                                         struct kvm_coalesced_mmio_zone *zone)
{
        int ret;
        struct kvm_coalesced_mmio_dev *dev;

        if (zone->pio != 1 && zone->pio != 0)
                return -EINVAL;

        dev = kzalloc(sizeof(struct kvm_coalesced_mmio_dev),
                      GFP_KERNEL_ACCOUNT);
        if (!dev)
                return -ENOMEM;

        kvm_iodevice_init(&dev->dev, &coalesced_mmio_ops);
        dev->kvm = kvm;
        dev->zone = *zone;

        mutex_lock(&kvm->slots_lock);
        ret = kvm_io_bus_register_dev(kvm,
                                zone->pio ? KVM_PIO_BUS : KVM_MMIO_BUS,
                                zone->addr, zone->size, &dev->dev);
        if (ret < 0)
                goto out_free_dev;
        list_add_tail(&dev->list, &kvm->coalesced_zones);
        mutex_unlock(&kvm->slots_lock);

        return 0;

out_free_dev:
        mutex_unlock(&kvm->slots_lock);
        kfree(dev);

        return ret;
}

int kvm_vm_ioctl_unregister_coalesced_mmio(struct kvm *kvm,
                                           struct kvm_coalesced_mmio_zone *zone)
{
        struct kvm_coalesced_mmio_dev *dev, *tmp;
        int r;

        if (zone->pio != 1 && zone->pio != 0)
                return -EINVAL;

        mutex_lock(&kvm->slots_lock);

        list_for_each_entry_safe(dev, tmp, &kvm->coalesced_zones, list) {
                if (zone->pio == dev->zone.pio &&
                    coalesced_mmio_in_range(dev, zone->addr, zone->size)) {
                        r = kvm_io_bus_unregister_dev(kvm,
                                zone->pio ? KVM_PIO_BUS : KVM_MMIO_BUS, &dev->dev);
                        /*
                         * On failure, unregister destroys all devices on the
                         * bus, including the target device. There's no need
                         * to restart the walk as there aren't any zones left.
                         */
                        if (r)
                                break;
                }
        }

        mutex_unlock(&kvm->slots_lock);

        /*
         * Ignore the result of kvm_io_bus_unregister_dev(), from userspace's
         * perspective, the coalesced MMIO is most definitely unregistered.
         */
        return 0;
}





















    9 















    9 



    9 

















    9 





    9 






    9 
    9 
    9 

    9 




    9 











    9 





    9 



    9 


    1 
    9 

    8 


    8 


    9 



















   10 
    1 





    9 

    9 

    9 






    9 
    9 





















   10 
   10 



   10 





   10 

   10 






   10 










   10 
    4 


    4 




    6 







   10 




   10 




    8 




   10 
















   10 


   11 




    9 
   10 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
// SPDX-License-Identifier: GPL-2.0-or-later
/* Direct I/O support.
 *
 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/export.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/uio.h>
#include <linux/sched/mm.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/netfs.h>
#include "internal.h"

static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq)
{
        struct netfs_io_request *rreq = subreq->rreq;
        size_t rsize;

        rsize = umin(subreq->len, rreq->io_streams[0].sreq_max_len);
        subreq->len = rsize;

        if (unlikely(rreq->io_streams[0].sreq_max_segs)) {
                size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize,
                                                rreq->io_streams[0].sreq_max_segs);

                if (limit < rsize) {
                        subreq->len = limit;
                        trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
                }
        }

        trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);

        subreq->io_iter        = rreq->buffer.iter;
        iov_iter_truncate(&subreq->io_iter, subreq->len);
        iov_iter_advance(&rreq->buffer.iter, subreq->len);
}

/*
 * Perform a read to a buffer from the server, slicing up the region to be read
 * according to the network rsize.
 */
static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
{
        struct netfs_io_stream *stream = &rreq->io_streams[0];
        unsigned long long start = rreq->start;
        ssize_t size = rreq->len;
        int ret = 0;

        do {
                struct netfs_io_subrequest *subreq;
                ssize_t slice;

                subreq = netfs_alloc_subrequest(rreq);
                if (!subreq) {
                        ret = -ENOMEM;
                        break;
                }

                subreq->source        = NETFS_DOWNLOAD_FROM_SERVER;
                subreq->start        = start;
                subreq->len        = size;

                __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);

                spin_lock(&rreq->lock);
                list_add_tail(&subreq->rreq_link, &stream->subrequests);
                if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
                        stream->front = subreq;
                        if (!stream->active) {
                                stream->collected_to = stream->front->start;
                                /* Store list pointers before active flag */
                                smp_store_release(&stream->active, true);
                        }
                }
                trace_netfs_sreq(subreq, netfs_sreq_trace_added);
                spin_unlock(&rreq->lock);

                netfs_stat(&netfs_n_rh_download);
                if (rreq->netfs_ops->prepare_read) {
                        ret = rreq->netfs_ops->prepare_read(subreq);
                        if (ret < 0) {
                                netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel);
                                break;
                        }
                }

                netfs_prepare_dio_read_iterator(subreq);
                slice = subreq->len;
                size -= slice;
                start += slice;
                rreq->submitted += slice;
                if (size <= 0) {
                        smp_wmb(); /* Write lists before ALL_QUEUED. */
                        set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
                }

                rreq->netfs_ops->issue_read(subreq);

                if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags))
                        netfs_wait_for_pause(rreq);
                if (test_bit(NETFS_RREQ_FAILED, &rreq->flags))
                        break;
                if (test_bit(NETFS_RREQ_BLOCKED, &rreq->flags) &&
                    test_bit(NETFS_RREQ_NONBLOCK, &rreq->flags))
                        break;
                cond_resched();
        } while (size > 0);

        if (unlikely(size > 0)) {
                smp_wmb(); /* Write lists before ALL_QUEUED. */
                set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
                netfs_wake_read_collector(rreq);
        }

        return ret;
}

/*
 * Perform a read to an application buffer, bypassing the pagecache and the
 * local disk cache.
 */
static ssize_t netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync)
{
        ssize_t ret;

        _enter("R=%x %llx-%llx",
               rreq->debug_id, rreq->start, rreq->start + rreq->len - 1);

        if (rreq->len == 0) {
                pr_err("Zero-sized read [R=%x]\n", rreq->debug_id);
                return -EIO;
        }

        // TODO: Use bounce buffer if requested

        inode_dio_begin(rreq->inode);

        ret = netfs_dispatch_unbuffered_reads(rreq);

        if (!rreq->submitted) {
                netfs_put_request(rreq, false, netfs_rreq_trace_put_no_submit);
                inode_dio_end(rreq->inode);
                ret = 0;
                goto out;
        }

        if (sync)
                ret = netfs_wait_for_read(rreq);
        else
                ret = -EIOCBQUEUED;
out:
        _leave(" = %zd", ret);
        return ret;
}

/**
 * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read
 * @iocb: The I/O control descriptor describing the read
 * @iter: The output buffer (also specifies read length)
 *
 * Perform an unbuffered I/O or direct I/O from the file in @iocb to the
 * output buffer.  No use is made of the pagecache.
 *
 * The caller must hold any appropriate locks.
 */
ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *iter)
{
        struct netfs_io_request *rreq;
        ssize_t ret;
        size_t orig_count = iov_iter_count(iter);
        bool sync = is_sync_kiocb(iocb);

        _enter("");

        if (!orig_count)
                return 0; /* Don't update atime */

        ret = kiocb_write_and_wait(iocb, orig_count);
        if (ret < 0)
                return ret;
        file_accessed(iocb->ki_filp);

        rreq = netfs_alloc_request(iocb->ki_filp->f_mapping, iocb->ki_filp,
                                   iocb->ki_pos, orig_count,
                                   NETFS_DIO_READ);
        if (IS_ERR(rreq))
                return PTR_ERR(rreq);

        netfs_stat(&netfs_n_rh_dio_read);
        trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_dio_read);

        /* If this is an async op, we have to keep track of the destination
         * buffer for ourselves as the caller's iterator will be trashed when
         * we return.
         *
         * In such a case, extract an iterator to represent as much of the the
         * output buffer as we can manage.  Note that the extraction might not
         * be able to allocate a sufficiently large bvec array and may shorten
         * the request.
         */
        if (user_backed_iter(iter)) {
                ret = netfs_extract_user_iter(iter, rreq->len, &rreq->buffer.iter, 0);
                if (ret < 0)
                        goto out;
                rreq->direct_bv = (struct bio_vec *)rreq->buffer.iter.bvec;
                rreq->direct_bv_count = ret;
                rreq->direct_bv_unpin = iov_iter_extract_will_pin(iter);
                rreq->len = iov_iter_count(&rreq->buffer.iter);
        } else {
                rreq->buffer.iter = *iter;
                rreq->len = orig_count;
                rreq->direct_bv_unpin = false;
                iov_iter_advance(iter, orig_count);
        }

        // TODO: Set up bounce buffer if needed

        if (!sync) {
                rreq->iocb = iocb;
                __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags);
        }

        ret = netfs_unbuffered_read(rreq, sync);
        if (ret < 0)
                goto out; /* May be -EIOCBQUEUED */
        if (sync) {
                // TODO: Copy from bounce buffer
                iocb->ki_pos += rreq->transferred;
                ret = rreq->transferred;
        }

out:
        netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
        if (ret > 0)
                orig_count -= ret;
        return ret;
}
EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked);

/**
 * netfs_unbuffered_read_iter - Perform an unbuffered or direct I/O read
 * @iocb: The I/O control descriptor describing the read
 * @iter: The output buffer (also specifies read length)
 *
 * Perform an unbuffered I/O or direct I/O from the file in @iocb to the
 * output buffer.  No use is made of the pagecache.
 */
ssize_t netfs_unbuffered_read_iter(struct kiocb *iocb, struct iov_iter *iter)
{
        struct inode *inode = file_inode(iocb->ki_filp);
        ssize_t ret;

        if (!iter->count)
                return 0; /* Don't update atime */

        ret = netfs_start_io_direct(inode);
        if (ret == 0) {
                ret = netfs_unbuffered_read_iter_locked(iocb, iter);
                netfs_end_io_direct(inode);
        }
        return ret;
}
EXPORT_SYMBOL(netfs_unbuffered_read_iter);





















   30 










    2 

    2 


    2 


    2 


    2 




    3 

    3 


    3 


    3 


    3 






   23 








    1 






    6 


    6 




    6 

    4 
    2 





    1 


    1 
    1 




    3 

    3 


    2 

    2 





    2 

    2 




    1 






    1 


    3 


    4 

    4 








    1 


    1 

   24 
    1 






   24 

   24 

   24 


















































































































































    1 


    1 


    1 


    1 



    1 






    1 








    1 

    1 


    1 




    1 

    1 




    1 
    1 












    1 




    1 

    1 
    1 
    1 



























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
// SPDX-License-Identifier: GPL-2.0

#include <linux/compat.h>
#include <linux/console.h>
#include <linux/fb.h>
#include <linux/fbcon.h>
#include <linux/major.h>

#include "fb_internal.h"

/*
 * We hold a reference to the fb_info in file->private_data,
 * but if the current registered fb has changed, we don't
 * actually want to use it.
 *
 * So look up the fb_info using the inode minor number,
 * and just verify it against the reference we have.
 */
static struct fb_info *file_fb_info(struct file *file)
{
        struct inode *inode = file_inode(file);
        int fbidx = iminor(inode);
        struct fb_info *info = registered_fb[fbidx];

        if (info != file->private_data)
                info = NULL;
        return info;
}

static ssize_t fb_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
        struct fb_info *info = file_fb_info(file);

        if (!info)
                return -ENODEV;

        if (fb_WARN_ON_ONCE(info, !info->fbops->fb_read))
                return -EINVAL;

        if (info->state != FBINFO_STATE_RUNNING)
                return -EPERM;

        return info->fbops->fb_read(info, buf, count, ppos);
}

static ssize_t fb_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
{
        struct fb_info *info = file_fb_info(file);

        if (!info)
                return -ENODEV;

        if (fb_WARN_ON_ONCE(info, !info->fbops->fb_write))
                return -EINVAL;

        if (info->state != FBINFO_STATE_RUNNING)
                return -EPERM;

        return info->fbops->fb_write(info, buf, count, ppos);
}

static long do_fb_ioctl(struct fb_info *info, unsigned int cmd,
                        unsigned long arg)
{
        const struct fb_ops *fb;
        struct fb_var_screeninfo var;
        struct fb_fix_screeninfo fix;
        struct fb_cmap cmap_from;
        struct fb_cmap_user cmap;
        void __user *argp = (void __user *)arg;
        long ret = 0;

        switch (cmd) {
        case FBIOGET_VSCREENINFO:
                lock_fb_info(info);
                var = info->var;
                unlock_fb_info(info);

                ret = copy_to_user(argp, &var, sizeof(var)) ? -EFAULT : 0;
                break;
        case FBIOPUT_VSCREENINFO:
                if (copy_from_user(&var, argp, sizeof(var)))
                        return -EFAULT;
                /* only for kernel-internal use */
                var.activate &= ~FB_ACTIVATE_KD_TEXT;
                console_lock();
                lock_fb_info(info);
                ret = fbcon_modechange_possible(info, &var);
                if (!ret)
                        ret = fb_set_var(info, &var);
                if (!ret)
                        fbcon_update_vcs(info, var.activate & FB_ACTIVATE_ALL);
                unlock_fb_info(info);
                console_unlock();
                if (!ret && copy_to_user(argp, &var, sizeof(var)))
                        ret = -EFAULT;
                break;
        case FBIOGET_FSCREENINFO:
                lock_fb_info(info);
                memcpy(&fix, &info->fix, sizeof(fix));
                if (info->flags & FBINFO_HIDE_SMEM_START)
                        fix.smem_start = 0;
                unlock_fb_info(info);

                ret = copy_to_user(argp, &fix, sizeof(fix)) ? -EFAULT : 0;
                break;
        case FBIOPUTCMAP:
                if (copy_from_user(&cmap, argp, sizeof(cmap)))
                        return -EFAULT;
                ret = fb_set_user_cmap(&cmap, info);
                break;
        case FBIOGETCMAP:
                if (copy_from_user(&cmap, argp, sizeof(cmap)))
                        return -EFAULT;
                lock_fb_info(info);
                cmap_from = info->cmap;
                unlock_fb_info(info);
                ret = fb_cmap_to_user(&cmap_from, &cmap);
                break;
        case FBIOPAN_DISPLAY:
                if (copy_from_user(&var, argp, sizeof(var)))
                        return -EFAULT;
                console_lock();
                lock_fb_info(info);
                ret = fb_pan_display(info, &var);
                unlock_fb_info(info);
                console_unlock();
                if (ret == 0 && copy_to_user(argp, &var, sizeof(var)))
                        return -EFAULT;
                break;
        case FBIO_CURSOR:
                ret = -EINVAL;
                break;
        case FBIOGET_CON2FBMAP:
                ret = fbcon_get_con2fb_map_ioctl(argp);
                break;
        case FBIOPUT_CON2FBMAP:
                ret = fbcon_set_con2fb_map_ioctl(argp);
                break;
        case FBIOBLANK:
                if (arg > FB_BLANK_POWERDOWN)
                        return -EINVAL;
                console_lock();
                lock_fb_info(info);
                ret = fb_blank(info, arg);
                /* might again call into fb_blank */
                fbcon_fb_blanked(info, arg);
                unlock_fb_info(info);
                console_unlock();
                break;
        default:
                lock_fb_info(info);
                fb = info->fbops;
                if (fb->fb_ioctl)
                        ret = fb->fb_ioctl(info, cmd, arg);
                else
                        ret = -ENOTTY;
                unlock_fb_info(info);
        }
        return ret;
}

static long fb_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
        struct fb_info *info = file_fb_info(file);

        if (!info)
                return -ENODEV;
        return do_fb_ioctl(info, cmd, arg);
}

#ifdef CONFIG_COMPAT
struct fb_fix_screeninfo32 {
        char                        id[16];
        compat_caddr_t                smem_start;
        u32                        smem_len;
        u32                        type;
        u32                        type_aux;
        u32                        visual;
        u16                        xpanstep;
        u16                        ypanstep;
        u16                        ywrapstep;
        u32                        line_length;
        compat_caddr_t                mmio_start;
        u32                        mmio_len;
        u32                        accel;
        u16                        reserved[3];
};

struct fb_cmap32 {
        u32                        start;
        u32                        len;
        compat_caddr_t        red;
        compat_caddr_t        green;
        compat_caddr_t        blue;
        compat_caddr_t        transp;
};

static int fb_getput_cmap(struct fb_info *info, unsigned int cmd,
                          unsigned long arg)
{
        struct fb_cmap32 cmap32;
        struct fb_cmap cmap_from;
        struct fb_cmap_user cmap;

        if (copy_from_user(&cmap32, compat_ptr(arg), sizeof(cmap32)))
                return -EFAULT;

        cmap = (struct fb_cmap_user) {
                .start        = cmap32.start,
                .len        = cmap32.len,
                .red        = compat_ptr(cmap32.red),
                .green        = compat_ptr(cmap32.green),
                .blue        = compat_ptr(cmap32.blue),
                .transp        = compat_ptr(cmap32.transp),
        };

        if (cmd == FBIOPUTCMAP)
                return fb_set_user_cmap(&cmap, info);

        lock_fb_info(info);
        cmap_from = info->cmap;
        unlock_fb_info(info);

        return fb_cmap_to_user(&cmap_from, &cmap);
}

static int do_fscreeninfo_to_user(struct fb_fix_screeninfo *fix,
                                  struct fb_fix_screeninfo32 __user *fix32)
{
        __u32 data;
        int err;

        err = copy_to_user(&fix32->id, &fix->id, sizeof(fix32->id));

        data = (__u32) (unsigned long) fix->smem_start;
        err |= put_user(data, &fix32->smem_start);

        err |= put_user(fix->smem_len, &fix32->smem_len);
        err |= put_user(fix->type, &fix32->type);
        err |= put_user(fix->type_aux, &fix32->type_aux);
        err |= put_user(fix->visual, &fix32->visual);
        err |= put_user(fix->xpanstep, &fix32->xpanstep);
        err |= put_user(fix->ypanstep, &fix32->ypanstep);
        err |= put_user(fix->ywrapstep, &fix32->ywrapstep);
        err |= put_user(fix->line_length, &fix32->line_length);

        data = (__u32) (unsigned long) fix->mmio_start;
        err |= put_user(data, &fix32->mmio_start);

        err |= put_user(fix->mmio_len, &fix32->mmio_len);
        err |= put_user(fix->accel, &fix32->accel);
        err |= copy_to_user(fix32->reserved, fix->reserved,
                            sizeof(fix->reserved));

        if (err)
                return -EFAULT;
        return 0;
}

static int fb_get_fscreeninfo(struct fb_info *info, unsigned int cmd,
                              unsigned long arg)
{
        struct fb_fix_screeninfo fix;

        lock_fb_info(info);
        fix = info->fix;
        if (info->flags & FBINFO_HIDE_SMEM_START)
                fix.smem_start = 0;
        unlock_fb_info(info);
        return do_fscreeninfo_to_user(&fix, compat_ptr(arg));
}

static long fb_compat_ioctl(struct file *file, unsigned int cmd,
                            unsigned long arg)
{
        struct fb_info *info = file_fb_info(file);
        const struct fb_ops *fb;
        long ret = -ENOIOCTLCMD;

        if (!info)
                return -ENODEV;
        fb = info->fbops;
        switch (cmd) {
        case FBIOGET_VSCREENINFO:
        case FBIOPUT_VSCREENINFO:
        case FBIOPAN_DISPLAY:
        case FBIOGET_CON2FBMAP:
        case FBIOPUT_CON2FBMAP:
                arg = (unsigned long) compat_ptr(arg);
                fallthrough;
        case FBIOBLANK:
                ret = do_fb_ioctl(info, cmd, arg);
                break;

        case FBIOGET_FSCREENINFO:
                ret = fb_get_fscreeninfo(info, cmd, arg);
                break;

        case FBIOGETCMAP:
        case FBIOPUTCMAP:
                ret = fb_getput_cmap(info, cmd, arg);
                break;

        default:
                if (fb->fb_compat_ioctl)
                        ret = fb->fb_compat_ioctl(info, cmd, arg);
                break;
        }
        return ret;
}
#endif

static int fb_mmap(struct file *file, struct vm_area_struct *vma)
{
        struct fb_info *info = file_fb_info(file);
        int res;

        if (!info)
                return -ENODEV;

        if (fb_WARN_ON_ONCE(info, !info->fbops->fb_mmap))
                return -ENODEV;

        mutex_lock(&info->mm_lock);
        res = info->fbops->fb_mmap(info, vma);
        mutex_unlock(&info->mm_lock);

        return res;
}

static int fb_open(struct inode *inode, struct file *file)
__acquires(&info->lock)
__releases(&info->lock)
{
        int fbidx = iminor(inode);
        struct fb_info *info;
        int res = 0;

        info = get_fb_info(fbidx);
        if (!info) {
                request_module("fb%d", fbidx);
                info = get_fb_info(fbidx);
                if (!info)
                        return -ENODEV;
        }
        if (IS_ERR(info))
                return PTR_ERR(info);

        lock_fb_info(info);
        if (!try_module_get(info->fbops->owner)) {
                res = -ENODEV;
                goto out;
        }
        file->private_data = info;
        if (info->fbops->fb_open) {
                res = info->fbops->fb_open(info, 1);
                if (res)
                        module_put(info->fbops->owner);
        }
#ifdef CONFIG_FB_DEFERRED_IO
        if (info->fbdefio)
                fb_deferred_io_open(info, inode, file);
#endif
out:
        unlock_fb_info(info);
        if (res)
                put_fb_info(info);
        return res;
}

static int fb_release(struct inode *inode, struct file *file)
__acquires(&info->lock)
__releases(&info->lock)
{
        struct fb_info * const info = file->private_data;

        lock_fb_info(info);
#if IS_ENABLED(CONFIG_FB_DEFERRED_IO)
        if (info->fbdefio)
                fb_deferred_io_release(info);
#endif
        if (info->fbops->fb_release)
                info->fbops->fb_release(info, 1);
        module_put(info->fbops->owner);
        unlock_fb_info(info);
        put_fb_info(info);
        return 0;
}

#if defined(CONFIG_FB_PROVIDE_GET_FB_UNMAPPED_AREA) && !defined(CONFIG_MMU)
static unsigned long get_fb_unmapped_area(struct file *filp,
                                   unsigned long addr, unsigned long len,
                                   unsigned long pgoff, unsigned long flags)
{
        struct fb_info * const info = filp->private_data;
        unsigned long fb_size = PAGE_ALIGN(info->fix.smem_len);

        if (pgoff > fb_size || len > fb_size - pgoff)
                return -EINVAL;

        return (unsigned long)info->screen_base + pgoff;
}
#endif

static const struct file_operations fb_fops = {
        .owner = THIS_MODULE,
        .read = fb_read,
        .write = fb_write,
        .unlocked_ioctl = fb_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl = fb_compat_ioctl,
#endif
        .mmap = fb_mmap,
        .open = fb_open,
        .release = fb_release,
#if defined(HAVE_ARCH_FB_UNMAPPED_AREA) || \
        (defined(CONFIG_FB_PROVIDE_GET_FB_UNMAPPED_AREA) && \
         !defined(CONFIG_MMU))
        .get_unmapped_area = get_fb_unmapped_area,
#endif
#ifdef CONFIG_FB_DEFERRED_IO
        .fsync = fb_deferred_io_fsync,
#endif
        .llseek = default_llseek,
};

int fb_register_chrdev(void)
{
        int ret;

        ret = register_chrdev(FB_MAJOR, "fb", &fb_fops);
        if (ret) {
                pr_err("Unable to get major %d for fb devs\n", FB_MAJOR);
                return ret;
        }

        return ret;
}

void fb_unregister_chrdev(void)
{
        unregister_chrdev(FB_MAJOR, "fb");
}























































    2 
















    1 

    1 





















   53 

    1 


    1 


   53 

    1 


    1 


   53 

    2 


    2 


   53 








   53 










   57 


   56 
    2 

   56 

























   55 
   57 
   57 




   57 



























   57 
   57 
   57 




   57 












    4 
    4 

    4 

    4 

























   55 













   56 



























































   52 



























   53 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
/*
 * Copyright (C) 2011-2013 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/errno.h>
#include <linux/export.h>
#include <linux/kernel.h>

#include <drm/drm_mode.h>
#include <drm/drm_print.h>
#include <drm/drm_rect.h>

/**
 * drm_rect_intersect - intersect two rectangles
 * @r1: first rectangle
 * @r2: second rectangle
 *
 * Calculate the intersection of rectangles @r1 and @r2.
 * @r1 will be overwritten with the intersection.
 *
 * RETURNS:
 * %true if rectangle @r1 is still visible after the operation,
 * %false otherwise.
 */
bool drm_rect_intersect(struct drm_rect *r1, const struct drm_rect *r2)
{
        r1->x1 = max(r1->x1, r2->x1);
        r1->y1 = max(r1->y1, r2->y1);
        r1->x2 = min(r1->x2, r2->x2);
        r1->y2 = min(r1->y2, r2->y2);

        return drm_rect_visible(r1);
}
EXPORT_SYMBOL(drm_rect_intersect);

static u32 clip_scaled(int src, int dst, int *clip)
{
        u64 tmp;

        if (dst == 0)
                return 0;

        /* Only clip what we have. Keeps the result bounded. */
        *clip = min(*clip, dst);

        tmp = mul_u32_u32(src, dst - *clip);

        /*
         * Round toward 1.0 when clipping so that we don't accidentally
         * change upscaling to downscaling or vice versa.
         */
        if (src < (dst << 16))
                return DIV_ROUND_UP_ULL(tmp, dst);
        else
                return DIV_ROUND_DOWN_ULL(tmp, dst);
}

/**
 * drm_rect_clip_scaled - perform a scaled clip operation
 * @src: source window rectangle
 * @dst: destination window rectangle
 * @clip: clip rectangle
 *
 * Clip rectangle @dst by rectangle @clip. Clip rectangle @src by
 * the corresponding amounts, retaining the vertical and horizontal scaling
 * factors from @src to @dst.
 *
 * RETURNS:
 * %true if rectangle @dst is still visible after being clipped,
 * %false otherwise.
 */
bool drm_rect_clip_scaled(struct drm_rect *src, struct drm_rect *dst,
                          const struct drm_rect *clip)
{
        int diff;

        diff = clip->x1 - dst->x1;
        if (diff > 0) {
                u32 new_src_w = clip_scaled(drm_rect_width(src),
                                            drm_rect_width(dst), &diff);

                src->x1 = src->x2 - new_src_w;
                dst->x1 += diff;
        }
        diff = clip->y1 - dst->y1;
        if (diff > 0) {
                u32 new_src_h = clip_scaled(drm_rect_height(src),
                                            drm_rect_height(dst), &diff);

                src->y1 = src->y2 - new_src_h;
                dst->y1 += diff;
        }
        diff = dst->x2 - clip->x2;
        if (diff > 0) {
                u32 new_src_w = clip_scaled(drm_rect_width(src),
                                            drm_rect_width(dst), &diff);

                src->x2 = src->x1 + new_src_w;
                dst->x2 -= diff;
        }
        diff = dst->y2 - clip->y2;
        if (diff > 0) {
                u32 new_src_h = clip_scaled(drm_rect_height(src),
                                            drm_rect_height(dst), &diff);

                src->y2 = src->y1 + new_src_h;
                dst->y2 -= diff;
        }

        return drm_rect_visible(dst);
}
EXPORT_SYMBOL(drm_rect_clip_scaled);

static int drm_calc_scale(int src, int dst)
{
        int scale = 0;

        if (WARN_ON(src < 0 || dst < 0))
                return -EINVAL;

        if (dst == 0)
                return 0;

        if (src > (dst << 16))
                return DIV_ROUND_UP(src, dst);
        else
                scale = src / dst;

        return scale;
}

/**
 * drm_rect_calc_hscale - calculate the horizontal scaling factor
 * @src: source window rectangle
 * @dst: destination window rectangle
 * @min_hscale: minimum allowed horizontal scaling factor
 * @max_hscale: maximum allowed horizontal scaling factor
 *
 * Calculate the horizontal scaling factor as
 * (@src width) / (@dst width).
 *
 * If the scale is below 1 << 16, round down. If the scale is above
 * 1 << 16, round up. This will calculate the scale with the most
 * pessimistic limit calculation.
 *
 * RETURNS:
 * The horizontal scaling factor, or errno of out of limits.
 */
int drm_rect_calc_hscale(const struct drm_rect *src,
                         const struct drm_rect *dst,
                         int min_hscale, int max_hscale)
{
        int src_w = drm_rect_width(src);
        int dst_w = drm_rect_width(dst);
        int hscale = drm_calc_scale(src_w, dst_w);

        if (hscale < 0 || dst_w == 0)
                return hscale;

        if (hscale < min_hscale || hscale > max_hscale)
                return -ERANGE;

        return hscale;
}
EXPORT_SYMBOL(drm_rect_calc_hscale);

/**
 * drm_rect_calc_vscale - calculate the vertical scaling factor
 * @src: source window rectangle
 * @dst: destination window rectangle
 * @min_vscale: minimum allowed vertical scaling factor
 * @max_vscale: maximum allowed vertical scaling factor
 *
 * Calculate the vertical scaling factor as
 * (@src height) / (@dst height).
 *
 * If the scale is below 1 << 16, round down. If the scale is above
 * 1 << 16, round up. This will calculate the scale with the most
 * pessimistic limit calculation.
 *
 * RETURNS:
 * The vertical scaling factor, or errno of out of limits.
 */
int drm_rect_calc_vscale(const struct drm_rect *src,
                         const struct drm_rect *dst,
                         int min_vscale, int max_vscale)
{
        int src_h = drm_rect_height(src);
        int dst_h = drm_rect_height(dst);
        int vscale = drm_calc_scale(src_h, dst_h);

        if (vscale < 0 || dst_h == 0)
                return vscale;

        if (vscale < min_vscale || vscale > max_vscale)
                return -ERANGE;

        return vscale;
}
EXPORT_SYMBOL(drm_rect_calc_vscale);

/**
 * drm_rect_debug_print - print the rectangle information
 * @prefix: prefix string
 * @r: rectangle to print
 * @fixed_point: rectangle is in 16.16 fixed point format
 */
void drm_rect_debug_print(const char *prefix, const struct drm_rect *r, bool fixed_point)
{
        if (fixed_point)
                DRM_DEBUG_KMS("%s" DRM_RECT_FP_FMT "\n", prefix, DRM_RECT_FP_ARG(r));
        else
                DRM_DEBUG_KMS("%s" DRM_RECT_FMT "\n", prefix, DRM_RECT_ARG(r));
}
EXPORT_SYMBOL(drm_rect_debug_print);

/**
 * drm_rect_rotate - Rotate the rectangle
 * @r: rectangle to be rotated
 * @width: Width of the coordinate space
 * @height: Height of the coordinate space
 * @rotation: Transformation to be applied
 *
 * Apply @rotation to the coordinates of rectangle @r.
 *
 * @width and @height combined with @rotation define
 * the location of the new origin.
 *
 * @width correcsponds to the horizontal and @height
 * to the vertical axis of the untransformed coordinate
 * space.
 */
void drm_rect_rotate(struct drm_rect *r,
                     int width, int height,
                     unsigned int rotation)
{
        struct drm_rect tmp;

        if (rotation & (DRM_MODE_REFLECT_X | DRM_MODE_REFLECT_Y)) {
                tmp = *r;

                if (rotation & DRM_MODE_REFLECT_X) {
                        r->x1 = width - tmp.x2;
                        r->x2 = width - tmp.x1;
                }

                if (rotation & DRM_MODE_REFLECT_Y) {
                        r->y1 = height - tmp.y2;
                        r->y2 = height - tmp.y1;
                }
        }

        switch (rotation & DRM_MODE_ROTATE_MASK) {
        case DRM_MODE_ROTATE_0:
                break;
        case DRM_MODE_ROTATE_90:
                tmp = *r;
                r->x1 = tmp.y1;
                r->x2 = tmp.y2;
                r->y1 = width - tmp.x2;
                r->y2 = width - tmp.x1;
                break;
        case DRM_MODE_ROTATE_180:
                tmp = *r;
                r->x1 = width - tmp.x2;
                r->x2 = width - tmp.x1;
                r->y1 = height - tmp.y2;
                r->y2 = height - tmp.y1;
                break;
        case DRM_MODE_ROTATE_270:
                tmp = *r;
                r->x1 = height - tmp.y2;
                r->x2 = height - tmp.y1;
                r->y1 = tmp.x1;
                r->y2 = tmp.x2;
                break;
        default:
                break;
        }
}
EXPORT_SYMBOL(drm_rect_rotate);

/**
 * drm_rect_rotate_inv - Inverse rotate the rectangle
 * @r: rectangle to be rotated
 * @width: Width of the coordinate space
 * @height: Height of the coordinate space
 * @rotation: Transformation whose inverse is to be applied
 *
 * Apply the inverse of @rotation to the coordinates
 * of rectangle @r.
 *
 * @width and @height combined with @rotation define
 * the location of the new origin.
 *
 * @width correcsponds to the horizontal and @height
 * to the vertical axis of the original untransformed
 * coordinate space, so that you never have to flip
 * them when doing a rotatation and its inverse.
 * That is, if you do ::
 *
 *     drm_rect_rotate(&r, width, height, rotation);
 *     drm_rect_rotate_inv(&r, width, height, rotation);
 *
 * you will always get back the original rectangle.
 */
void drm_rect_rotate_inv(struct drm_rect *r,
                         int width, int height,
                         unsigned int rotation)
{
        struct drm_rect tmp;

        switch (rotation & DRM_MODE_ROTATE_MASK) {
        case DRM_MODE_ROTATE_0:
                break;
        case DRM_MODE_ROTATE_90:
                tmp = *r;
                r->x1 = width - tmp.y2;
                r->x2 = width - tmp.y1;
                r->y1 = tmp.x1;
                r->y2 = tmp.x2;
                break;
        case DRM_MODE_ROTATE_180:
                tmp = *r;
                r->x1 = width - tmp.x2;
                r->x2 = width - tmp.x1;
                r->y1 = height - tmp.y2;
                r->y2 = height - tmp.y1;
                break;
        case DRM_MODE_ROTATE_270:
                tmp = *r;
                r->x1 = tmp.y1;
                r->x2 = tmp.y2;
                r->y1 = height - tmp.x2;
                r->y2 = height - tmp.x1;
                break;
        default:
                break;
        }

        if (rotation & (DRM_MODE_REFLECT_X | DRM_MODE_REFLECT_Y)) {
                tmp = *r;

                if (rotation & DRM_MODE_REFLECT_X) {
                        r->x1 = width - tmp.x2;
                        r->x2 = width - tmp.x1;
                }

                if (rotation & DRM_MODE_REFLECT_Y) {
                        r->y1 = height - tmp.y2;
                        r->y2 = height - tmp.y1;
                }
        }
}
EXPORT_SYMBOL(drm_rect_rotate_inv);
























































    1 
    1 



    1 






    1 

    1 







    1 
    1 


    1 
    1 

    1 
    1 
















    5 









    2 

    4 

    4 


    5 




    5 




    5 



    5 









    1 




    1 


    1 

    1 




    3 


    1 

    3 
    3 

    3 











    4 

    3 
    1 
    1 
    1 
    1 
    1 
    4 

    2 




    4 



    3 









    3 









    8 






    8 

    2 
    1 
    1 




    8 
    2 
    2 



    8 




    1 





    2 










   13 



    5 


   13 
    1 
    1 




   13 
    3 



   13 
    1 




   13 


    3 

   13 
    5 

   13 
   11 
    7 
   13 















   13 




    3 
   13 
    3 

   13 
    5 

   13 
    9 
   13 































    1 








    1 




    1 




    1 




    1 
    1 





    1 







    1 




   20 



   20 
    7 

   20 















































































   13 


    7 
    5 
    7 
    3 





   14 





    2 

    2 














    2 


    2 

    2 

















   20 











    5 

   20 






   20 








   20 

   20 






   20 


   20 
   20 





   20 
   20 
   14 
   20 

    6 







   20 

   20 




   20 

   20 









   15 
   15 
    1 







   14 





   27 



   27 








   27 










    8 





    1 
    1 







    2 









   38 



   37 


















   38 
















































   36 


















































































































































































































































































































   13 



   13 




   26 



















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
// SPDX-License-Identifier: GPL-2.0-or-later
/* -*- linux-c -*-
 * INET                802.1Q VLAN
 *                Ethernet-type device handling.
 *
 * Authors:        Ben Greear <greearb@candelatech.com>
 *              Please send support related email to: netdev@vger.kernel.org
 *              VLAN Home Page: http://www.candelatech.com/~greear/vlan.html
 *
 * Fixes:       Mar 22 2001: Martin Bokaemper <mbokaemper@unispherenetworks.com>
 *                - reset skb->pkt_type on incoming packets when MAC was changed
 *                - see that changed MAC is saddr for outgoing packets
 *              Oct 20, 2001:  Ard van Breeman:
 *                - Fix MC-list, finally.
 *                - Flush MC-list on VLAN destroy.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/net_tstamp.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/phy.h>
#include <net/arp.h>
#include <net/macsec.h>

#include "vlan.h"
#include "vlanproc.h"
#include <linux/if_vlan.h>
#include <linux/netpoll.h>

/*
 *        Create the VLAN header for an arbitrary protocol layer
 *
 *        saddr=NULL        means use device source address
 *        daddr=NULL        means leave destination address (eg unresolved arp)
 *
 *  This is called when the SKB is moving down the stack towards the
 *  physical devices.
 */
static int vlan_dev_hard_header(struct sk_buff *skb, struct net_device *dev,
                                unsigned short type,
                                const void *daddr, const void *saddr,
                                unsigned int len)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        struct vlan_hdr *vhdr;
        unsigned int vhdrlen = 0;
        u16 vlan_tci = 0;
        int rc;

        if (!(vlan->flags & VLAN_FLAG_REORDER_HDR)) {
                vhdr = skb_push(skb, VLAN_HLEN);

                vlan_tci = vlan->vlan_id;
                vlan_tci |= vlan_dev_get_egress_qos_mask(dev, skb->priority);
                vhdr->h_vlan_TCI = htons(vlan_tci);

                /*
                 *  Set the protocol type. For a packet of type ETH_P_802_3/2 we
                 *  put the length in here instead.
                 */
                if (type != ETH_P_802_3 && type != ETH_P_802_2)
                        vhdr->h_vlan_encapsulated_proto = htons(type);
                else
                        vhdr->h_vlan_encapsulated_proto = htons(len);

                skb->protocol = vlan->vlan_proto;
                type = ntohs(vlan->vlan_proto);
                vhdrlen = VLAN_HLEN;
        }

        /* Before delegating work to the lower layer, enter our MAC-address */
        if (saddr == NULL)
                saddr = dev->dev_addr;

        /* Now make the underlying real hard header */
        dev = vlan->real_dev;
        rc = dev_hard_header(skb, dev, type, daddr, saddr, len + vhdrlen);
        if (rc > 0)
                rc += vhdrlen;
        return rc;
}

static inline netdev_tx_t vlan_netpoll_send_skb(struct vlan_dev_priv *vlan, struct sk_buff *skb)
{
#ifdef CONFIG_NET_POLL_CONTROLLER
        return netpoll_send_skb(vlan->netpoll, skb);
#else
        BUG();
        return NETDEV_TX_OK;
#endif
}

static netdev_tx_t vlan_dev_hard_start_xmit(struct sk_buff *skb,
                                            struct net_device *dev)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        struct vlan_ethhdr *veth = (struct vlan_ethhdr *)(skb->data);
        unsigned int len;
        int ret;

        /* Handle non-VLAN frames if they are sent to us, for example by DHCP.
         *
         * NOTE: THIS ASSUMES DIX ETHERNET, SPECIFICALLY NOT SUPPORTING
         * OTHER THINGS LIKE FDDI/TokenRing/802.3 SNAPs...
         */
        if (vlan->flags & VLAN_FLAG_REORDER_HDR ||
            veth->h_vlan_proto != vlan->vlan_proto) {
                u16 vlan_tci;
                vlan_tci = vlan->vlan_id;
                vlan_tci |= vlan_dev_get_egress_qos_mask(dev, skb->priority);
                __vlan_hwaccel_put_tag(skb, vlan->vlan_proto, vlan_tci);
        }

        skb->dev = vlan->real_dev;
        len = skb->len;
        if (unlikely(netpoll_tx_running(dev)))
                return vlan_netpoll_send_skb(vlan, skb);

        ret = dev_queue_xmit(skb);

        if (likely(ret == NET_XMIT_SUCCESS || ret == NET_XMIT_CN)) {
                struct vlan_pcpu_stats *stats;

                stats = this_cpu_ptr(vlan->vlan_pcpu_stats);
                u64_stats_update_begin(&stats->syncp);
                u64_stats_inc(&stats->tx_packets);
                u64_stats_add(&stats->tx_bytes, len);
                u64_stats_update_end(&stats->syncp);
        } else {
                this_cpu_inc(vlan->vlan_pcpu_stats->tx_dropped);
        }

        return ret;
}

static int vlan_dev_change_mtu(struct net_device *dev, int new_mtu)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        unsigned int max_mtu = real_dev->mtu;

        if (netif_reduces_vlan_mtu(real_dev))
                max_mtu -= VLAN_HLEN;
        if (max_mtu < new_mtu)
                return -ERANGE;

        WRITE_ONCE(dev->mtu, new_mtu);

        return 0;
}

void vlan_dev_set_ingress_priority(const struct net_device *dev,
                                   u32 skb_prio, u16 vlan_prio)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);

        if (vlan->ingress_priority_map[vlan_prio & 0x7] && !skb_prio)
                vlan->nr_ingress_mappings--;
        else if (!vlan->ingress_priority_map[vlan_prio & 0x7] && skb_prio)
                vlan->nr_ingress_mappings++;

        vlan->ingress_priority_map[vlan_prio & 0x7] = skb_prio;
}

int vlan_dev_set_egress_priority(const struct net_device *dev,
                                 u32 skb_prio, u16 vlan_prio)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        struct vlan_priority_tci_mapping *mp = NULL;
        struct vlan_priority_tci_mapping *np;
        u32 vlan_qos = (vlan_prio << VLAN_PRIO_SHIFT) & VLAN_PRIO_MASK;

        /* See if a priority mapping exists.. */
        mp = vlan->egress_priority_map[skb_prio & 0xF];
        while (mp) {
                if (mp->priority == skb_prio) {
                        if (mp->vlan_qos && !vlan_qos)
                                vlan->nr_egress_mappings--;
                        else if (!mp->vlan_qos && vlan_qos)
                                vlan->nr_egress_mappings++;
                        mp->vlan_qos = vlan_qos;
                        return 0;
                }
                mp = mp->next;
        }

        /* Create a new mapping then. */
        mp = vlan->egress_priority_map[skb_prio & 0xF];
        np = kmalloc(sizeof(struct vlan_priority_tci_mapping), GFP_KERNEL);
        if (!np)
                return -ENOBUFS;

        np->next = mp;
        np->priority = skb_prio;
        np->vlan_qos = vlan_qos;
        /* Before inserting this element in hash table, make sure all its fields
         * are committed to memory.
         * coupled with smp_rmb() in vlan_dev_get_egress_qos_mask()
         */
        smp_wmb();
        vlan->egress_priority_map[skb_prio & 0xF] = np;
        if (vlan_qos)
                vlan->nr_egress_mappings++;
        return 0;
}

/* Flags are defined in the vlan_flags enum in
 * include/uapi/linux/if_vlan.h file.
 */
int vlan_dev_change_flags(const struct net_device *dev, u32 flags, u32 mask)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        u32 old_flags = vlan->flags;

        if (mask & ~(VLAN_FLAG_REORDER_HDR | VLAN_FLAG_GVRP |
                     VLAN_FLAG_LOOSE_BINDING | VLAN_FLAG_MVRP |
                     VLAN_FLAG_BRIDGE_BINDING))
                return -EINVAL;

        vlan->flags = (old_flags & ~mask) | (flags & mask);

        if (netif_running(dev) && (vlan->flags ^ old_flags) & VLAN_FLAG_GVRP) {
                if (vlan->flags & VLAN_FLAG_GVRP)
                        vlan_gvrp_request_join(dev);
                else
                        vlan_gvrp_request_leave(dev);
        }

        if (netif_running(dev) && (vlan->flags ^ old_flags) & VLAN_FLAG_MVRP) {
                if (vlan->flags & VLAN_FLAG_MVRP)
                        vlan_mvrp_request_join(dev);
                else
                        vlan_mvrp_request_leave(dev);
        }
        return 0;
}

void vlan_dev_get_realdev_name(const struct net_device *dev, char *result, size_t size)
{
        strscpy_pad(result, vlan_dev_priv(dev)->real_dev->name, size);
}

bool vlan_dev_inherit_address(struct net_device *dev,
                              struct net_device *real_dev)
{
        if (dev->addr_assign_type != NET_ADDR_STOLEN)
                return false;

        eth_hw_addr_set(dev, real_dev->dev_addr);
        call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
        return true;
}

static int vlan_dev_open(struct net_device *dev)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        struct net_device *real_dev = vlan->real_dev;
        int err;

        if (!(real_dev->flags & IFF_UP) &&
            !(vlan->flags & VLAN_FLAG_LOOSE_BINDING))
                return -ENETDOWN;

        if (!ether_addr_equal(dev->dev_addr, real_dev->dev_addr) &&
            !vlan_dev_inherit_address(dev, real_dev)) {
                err = dev_uc_add(real_dev, dev->dev_addr);
                if (err < 0)
                        goto out;
        }

        if (dev->flags & IFF_ALLMULTI) {
                err = dev_set_allmulti(real_dev, 1);
                if (err < 0)
                        goto del_unicast;
        }
        if (dev->flags & IFF_PROMISC) {
                err = dev_set_promiscuity(real_dev, 1);
                if (err < 0)
                        goto clear_allmulti;
        }

        ether_addr_copy(vlan->real_dev_addr, real_dev->dev_addr);

        if (vlan->flags & VLAN_FLAG_GVRP)
                vlan_gvrp_request_join(dev);

        if (vlan->flags & VLAN_FLAG_MVRP)
                vlan_mvrp_request_join(dev);

        if (netif_carrier_ok(real_dev) &&
            !(vlan->flags & VLAN_FLAG_BRIDGE_BINDING))
                netif_carrier_on(dev);
        return 0;

clear_allmulti:
        if (dev->flags & IFF_ALLMULTI)
                dev_set_allmulti(real_dev, -1);
del_unicast:
        if (!ether_addr_equal(dev->dev_addr, real_dev->dev_addr))
                dev_uc_del(real_dev, dev->dev_addr);
out:
        netif_carrier_off(dev);
        return err;
}

static int vlan_dev_stop(struct net_device *dev)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        struct net_device *real_dev = vlan->real_dev;

        dev_mc_unsync(real_dev, dev);
        dev_uc_unsync(real_dev, dev);
        if (dev->flags & IFF_ALLMULTI)
                dev_set_allmulti(real_dev, -1);
        if (dev->flags & IFF_PROMISC)
                dev_set_promiscuity(real_dev, -1);

        if (!ether_addr_equal(dev->dev_addr, real_dev->dev_addr))
                dev_uc_del(real_dev, dev->dev_addr);

        if (!(vlan->flags & VLAN_FLAG_BRIDGE_BINDING))
                netif_carrier_off(dev);
        return 0;
}

static int vlan_dev_set_mac_address(struct net_device *dev, void *p)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        struct sockaddr *addr = p;
        int err;

        if (!is_valid_ether_addr(addr->sa_data))
                return -EADDRNOTAVAIL;

        if (!(dev->flags & IFF_UP))
                goto out;

        if (!ether_addr_equal(addr->sa_data, real_dev->dev_addr)) {
                err = dev_uc_add(real_dev, addr->sa_data);
                if (err < 0)
                        return err;
        }

        if (!ether_addr_equal(dev->dev_addr, real_dev->dev_addr))
                dev_uc_del(real_dev, dev->dev_addr);

out:
        eth_hw_addr_set(dev, addr->sa_data);
        return 0;
}

static int vlan_hwtstamp_get(struct net_device *dev,
                             struct kernel_hwtstamp_config *cfg)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;

        return generic_hwtstamp_get_lower(real_dev, cfg);
}

static int vlan_hwtstamp_set(struct net_device *dev,
                             struct kernel_hwtstamp_config *cfg,
                             struct netlink_ext_ack *extack)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;

        if (!net_eq(dev_net(dev), dev_net(real_dev)))
                return -EOPNOTSUPP;

        return generic_hwtstamp_set_lower(real_dev, cfg, extack);
}

static int vlan_dev_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        const struct net_device_ops *ops = real_dev->netdev_ops;
        struct ifreq ifrr;
        int err = -EOPNOTSUPP;

        strscpy_pad(ifrr.ifr_name, real_dev->name, IFNAMSIZ);
        ifrr.ifr_ifru = ifr->ifr_ifru;

        switch (cmd) {
        case SIOCGMIIPHY:
        case SIOCGMIIREG:
        case SIOCSMIIREG:
                if (netif_device_present(real_dev) && ops->ndo_eth_ioctl)
                        err = ops->ndo_eth_ioctl(real_dev, &ifrr, cmd);
                break;
        }

        if (!err)
                ifr->ifr_ifru = ifrr.ifr_ifru;

        return err;
}

static int vlan_dev_neigh_setup(struct net_device *dev, struct neigh_parms *pa)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        const struct net_device_ops *ops = real_dev->netdev_ops;
        int err = 0;

        if (netif_device_present(real_dev) && ops->ndo_neigh_setup)
                err = ops->ndo_neigh_setup(real_dev, pa);

        return err;
}

#if IS_ENABLED(CONFIG_FCOE)
static int vlan_dev_fcoe_ddp_setup(struct net_device *dev, u16 xid,
                                   struct scatterlist *sgl, unsigned int sgc)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        const struct net_device_ops *ops = real_dev->netdev_ops;
        int rc = 0;

        if (ops->ndo_fcoe_ddp_setup)
                rc = ops->ndo_fcoe_ddp_setup(real_dev, xid, sgl, sgc);

        return rc;
}

static int vlan_dev_fcoe_ddp_done(struct net_device *dev, u16 xid)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        const struct net_device_ops *ops = real_dev->netdev_ops;
        int len = 0;

        if (ops->ndo_fcoe_ddp_done)
                len = ops->ndo_fcoe_ddp_done(real_dev, xid);

        return len;
}

static int vlan_dev_fcoe_enable(struct net_device *dev)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        const struct net_device_ops *ops = real_dev->netdev_ops;
        int rc = -EINVAL;

        if (ops->ndo_fcoe_enable)
                rc = ops->ndo_fcoe_enable(real_dev);
        return rc;
}

static int vlan_dev_fcoe_disable(struct net_device *dev)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        const struct net_device_ops *ops = real_dev->netdev_ops;
        int rc = -EINVAL;

        if (ops->ndo_fcoe_disable)
                rc = ops->ndo_fcoe_disable(real_dev);
        return rc;
}

static int vlan_dev_fcoe_ddp_target(struct net_device *dev, u16 xid,
                                    struct scatterlist *sgl, unsigned int sgc)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        const struct net_device_ops *ops = real_dev->netdev_ops;
        int rc = 0;

        if (ops->ndo_fcoe_ddp_target)
                rc = ops->ndo_fcoe_ddp_target(real_dev, xid, sgl, sgc);

        return rc;
}
#endif

#ifdef NETDEV_FCOE_WWNN
static int vlan_dev_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        const struct net_device_ops *ops = real_dev->netdev_ops;
        int rc = -EINVAL;

        if (ops->ndo_fcoe_get_wwn)
                rc = ops->ndo_fcoe_get_wwn(real_dev, wwn, type);
        return rc;
}
#endif

static void vlan_dev_change_rx_flags(struct net_device *dev, int change)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;

        if (dev->flags & IFF_UP) {
                if (change & IFF_ALLMULTI)
                        dev_set_allmulti(real_dev, dev->flags & IFF_ALLMULTI ? 1 : -1);
                if (change & IFF_PROMISC)
                        dev_set_promiscuity(real_dev, dev->flags & IFF_PROMISC ? 1 : -1);
        }
}

static void vlan_dev_set_rx_mode(struct net_device *vlan_dev)
{
        dev_mc_sync(vlan_dev_priv(vlan_dev)->real_dev, vlan_dev);
        dev_uc_sync(vlan_dev_priv(vlan_dev)->real_dev, vlan_dev);
}

static __be16 vlan_parse_protocol(const struct sk_buff *skb)
{
        struct vlan_ethhdr *veth = (struct vlan_ethhdr *)(skb->data);

        return __vlan_get_protocol(skb, veth->h_vlan_proto, NULL);
}

static const struct header_ops vlan_header_ops = {
        .create         = vlan_dev_hard_header,
        .parse         = eth_header_parse,
        .parse_protocol = vlan_parse_protocol,
};

static int vlan_passthru_hard_header(struct sk_buff *skb, struct net_device *dev,
                                     unsigned short type,
                                     const void *daddr, const void *saddr,
                                     unsigned int len)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        struct net_device *real_dev = vlan->real_dev;

        if (saddr == NULL)
                saddr = dev->dev_addr;

        return dev_hard_header(skb, real_dev, type, daddr, saddr, len);
}

static const struct header_ops vlan_passthru_header_ops = {
        .create         = vlan_passthru_hard_header,
        .parse         = eth_header_parse,
        .parse_protocol = vlan_parse_protocol,
};

static const struct device_type vlan_type = {
        .name        = "vlan",
};

static const struct net_device_ops vlan_netdev_ops;

static int vlan_dev_init(struct net_device *dev)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        struct net_device *real_dev = vlan->real_dev;

        netif_carrier_off(dev);

        /* IFF_BROADCAST|IFF_MULTICAST; ??? */
        dev->flags  = real_dev->flags & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI |
                                          IFF_MASTER | IFF_SLAVE);
        dev->state  = (real_dev->state & ((1<<__LINK_STATE_NOCARRIER) |
                                          (1<<__LINK_STATE_DORMANT))) |
                      (1<<__LINK_STATE_PRESENT);

        if (vlan->flags & VLAN_FLAG_BRIDGE_BINDING)
                dev->state |= (1 << __LINK_STATE_NOCARRIER);

        dev->hw_features = NETIF_F_HW_CSUM | NETIF_F_SG |
                           NETIF_F_FRAGLIST | NETIF_F_GSO_SOFTWARE |
                           NETIF_F_GSO_ENCAP_ALL |
                           NETIF_F_HIGHDMA | NETIF_F_SCTP_CRC |
                           NETIF_F_FCOE_CRC | NETIF_F_FSO;

        if (real_dev->vlan_features & NETIF_F_HW_MACSEC)
                dev->hw_features |= NETIF_F_HW_MACSEC;

        dev->features |= dev->hw_features;
        dev->lltx = true;
        dev->fcoe_mtu = true;
        netif_inherit_tso_max(dev, real_dev);
        if (dev->features & NETIF_F_VLAN_FEATURES)
                netdev_warn(real_dev, "VLAN features are set incorrectly.  Q-in-Q configurations may not work correctly.\n");

        dev->vlan_features = real_dev->vlan_features &
                             ~(NETIF_F_FCOE_CRC | NETIF_F_FSO);
        dev->hw_enc_features = vlan_tnl_features(real_dev);
        dev->mpls_features = real_dev->mpls_features;

        /* ipv6 shared card related stuff */
        dev->dev_id = real_dev->dev_id;

        if (is_zero_ether_addr(dev->dev_addr)) {
                eth_hw_addr_set(dev, real_dev->dev_addr);
                dev->addr_assign_type = NET_ADDR_STOLEN;
        }
        if (is_zero_ether_addr(dev->broadcast))
                memcpy(dev->broadcast, real_dev->broadcast, dev->addr_len);

#if IS_ENABLED(CONFIG_FCOE)
        dev->fcoe_ddp_xid = real_dev->fcoe_ddp_xid;
#endif

        dev->needed_headroom = real_dev->needed_headroom;
        if (vlan_hw_offload_capable(real_dev->features, vlan->vlan_proto)) {
                dev->header_ops      = &vlan_passthru_header_ops;
                dev->hard_header_len = real_dev->hard_header_len;
        } else {
                dev->header_ops      = &vlan_header_ops;
                dev->hard_header_len = real_dev->hard_header_len + VLAN_HLEN;
        }

        dev->netdev_ops = &vlan_netdev_ops;

        SET_NETDEV_DEVTYPE(dev, &vlan_type);

        netdev_lockdep_set_classes(dev);

        vlan->vlan_pcpu_stats = netdev_alloc_pcpu_stats(struct vlan_pcpu_stats);
        if (!vlan->vlan_pcpu_stats)
                return -ENOMEM;

        /* Get vlan's reference to real_dev */
        netdev_hold(real_dev, &vlan->dev_tracker, GFP_KERNEL);

        return 0;
}

/* Note: this function might be called multiple times for the same device. */
void vlan_dev_free_egress_priority(const struct net_device *dev)
{
        struct vlan_priority_tci_mapping *pm;
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        int i;

        for (i = 0; i < ARRAY_SIZE(vlan->egress_priority_map); i++) {
                while ((pm = vlan->egress_priority_map[i]) != NULL) {
                        vlan->egress_priority_map[i] = pm->next;
                        kfree(pm);
                }
        }
}

static void vlan_dev_uninit(struct net_device *dev)
{
        vlan_dev_free_egress_priority(dev);
}

static netdev_features_t vlan_dev_fix_features(struct net_device *dev,
        netdev_features_t features)
{
        struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
        netdev_features_t old_features = features;
        netdev_features_t lower_features;

        lower_features = netdev_intersect_features((real_dev->vlan_features |
                                                    NETIF_F_RXCSUM),
                                                   real_dev->features);

        /* Add HW_CSUM setting to preserve user ability to control
         * checksum offload on the vlan device.
         */
        if (lower_features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
                lower_features |= NETIF_F_HW_CSUM;
        features = netdev_intersect_features(features, lower_features);
        features |= old_features & (NETIF_F_SOFT_FEATURES | NETIF_F_GSO_SOFTWARE);

        return features;
}

static int vlan_ethtool_get_link_ksettings(struct net_device *dev,
                                           struct ethtool_link_ksettings *cmd)
{
        const struct vlan_dev_priv *vlan = vlan_dev_priv(dev);

        return __ethtool_get_link_ksettings(vlan->real_dev, cmd);
}

static void vlan_ethtool_get_drvinfo(struct net_device *dev,
                                     struct ethtool_drvinfo *info)
{
        strscpy(info->driver, vlan_fullname, sizeof(info->driver));
        strscpy(info->version, vlan_version, sizeof(info->version));
        strscpy(info->fw_version, "N/A", sizeof(info->fw_version));
}

static int vlan_ethtool_get_ts_info(struct net_device *dev,
                                    struct kernel_ethtool_ts_info *info)
{
        const struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        return ethtool_get_ts_info_by_layer(vlan->real_dev, info);
}

static void vlan_dev_get_stats64(struct net_device *dev,
                                 struct rtnl_link_stats64 *stats)
{
        struct vlan_pcpu_stats *p;
        u32 rx_errors = 0, tx_dropped = 0;
        int i;

        for_each_possible_cpu(i) {
                u64 rxpackets, rxbytes, rxmulticast, txpackets, txbytes;
                unsigned int start;

                p = per_cpu_ptr(vlan_dev_priv(dev)->vlan_pcpu_stats, i);
                do {
                        start = u64_stats_fetch_begin(&p->syncp);
                        rxpackets        = u64_stats_read(&p->rx_packets);
                        rxbytes                = u64_stats_read(&p->rx_bytes);
                        rxmulticast        = u64_stats_read(&p->rx_multicast);
                        txpackets        = u64_stats_read(&p->tx_packets);
                        txbytes                = u64_stats_read(&p->tx_bytes);
                } while (u64_stats_fetch_retry(&p->syncp, start));

                stats->rx_packets        += rxpackets;
                stats->rx_bytes                += rxbytes;
                stats->multicast        += rxmulticast;
                stats->tx_packets        += txpackets;
                stats->tx_bytes                += txbytes;
                /* rx_errors & tx_dropped are u32 */
                rx_errors        += READ_ONCE(p->rx_errors);
                tx_dropped        += READ_ONCE(p->tx_dropped);
        }
        stats->rx_errors  = rx_errors;
        stats->tx_dropped = tx_dropped;
}

#ifdef CONFIG_NET_POLL_CONTROLLER
static void vlan_dev_poll_controller(struct net_device *dev)
{
        return;
}

static int vlan_dev_netpoll_setup(struct net_device *dev)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
        struct net_device *real_dev = vlan->real_dev;
        struct netpoll *netpoll;
        int err = 0;

        netpoll = kzalloc(sizeof(*netpoll), GFP_KERNEL);
        err = -ENOMEM;
        if (!netpoll)
                goto out;

        err = __netpoll_setup(netpoll, real_dev);
        if (err) {
                kfree(netpoll);
                goto out;
        }

        vlan->netpoll = netpoll;

out:
        return err;
}

static void vlan_dev_netpoll_cleanup(struct net_device *dev)
{
        struct vlan_dev_priv *vlan= vlan_dev_priv(dev);
        struct netpoll *netpoll = vlan->netpoll;

        if (!netpoll)
                return;

        vlan->netpoll = NULL;
        __netpoll_free(netpoll);
}
#endif /* CONFIG_NET_POLL_CONTROLLER */

static int vlan_dev_get_iflink(const struct net_device *dev)
{
        const struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;

        return READ_ONCE(real_dev->ifindex);
}

static int vlan_dev_fill_forward_path(struct net_device_path_ctx *ctx,
                                      struct net_device_path *path)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(ctx->dev);

        path->type = DEV_PATH_VLAN;
        path->encap.id = vlan->vlan_id;
        path->encap.proto = vlan->vlan_proto;
        path->dev = ctx->dev;
        ctx->dev = vlan->real_dev;
        if (ctx->num_vlans >= ARRAY_SIZE(ctx->vlan))
                return -ENOSPC;

        ctx->vlan[ctx->num_vlans].id = vlan->vlan_id;
        ctx->vlan[ctx->num_vlans].proto = vlan->vlan_proto;
        ctx->num_vlans++;

        return 0;
}

#if IS_ENABLED(CONFIG_MACSEC)

static const struct macsec_ops *vlan_get_macsec_ops(const struct macsec_context *ctx)
{
        return vlan_dev_priv(ctx->netdev)->real_dev->macsec_ops;
}

static int vlan_macsec_offload(int (* const func)(struct macsec_context *),
                               struct macsec_context *ctx)
{
        if (unlikely(!func))
                return 0;

        return (*func)(ctx);
}

static int vlan_macsec_dev_open(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_dev_open, ctx);
}

static int vlan_macsec_dev_stop(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_dev_stop, ctx);
}

static int vlan_macsec_add_secy(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_add_secy, ctx);
}

static int vlan_macsec_upd_secy(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_upd_secy, ctx);
}

static int vlan_macsec_del_secy(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_del_secy, ctx);
}

static int vlan_macsec_add_rxsc(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_add_rxsc, ctx);
}

static int vlan_macsec_upd_rxsc(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_upd_rxsc, ctx);
}

static int vlan_macsec_del_rxsc(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_del_rxsc, ctx);
}

static int vlan_macsec_add_rxsa(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_add_rxsa, ctx);
}

static int vlan_macsec_upd_rxsa(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_upd_rxsa, ctx);
}

static int vlan_macsec_del_rxsa(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_del_rxsa, ctx);
}

static int vlan_macsec_add_txsa(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_add_txsa, ctx);
}

static int vlan_macsec_upd_txsa(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_upd_txsa, ctx);
}

static int vlan_macsec_del_txsa(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_del_txsa, ctx);
}

static int vlan_macsec_get_dev_stats(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_get_dev_stats, ctx);
}

static int vlan_macsec_get_tx_sc_stats(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_get_tx_sc_stats, ctx);
}

static int vlan_macsec_get_tx_sa_stats(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_get_tx_sa_stats, ctx);
}

static int vlan_macsec_get_rx_sc_stats(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_get_rx_sc_stats, ctx);
}

static int vlan_macsec_get_rx_sa_stats(struct macsec_context *ctx)
{
        const struct macsec_ops *ops = vlan_get_macsec_ops(ctx);

        if (!ops)
                return -EOPNOTSUPP;

        return vlan_macsec_offload(ops->mdo_get_rx_sa_stats, ctx);
}

static const struct macsec_ops macsec_offload_ops = {
        /* Device wide */
        .mdo_dev_open = vlan_macsec_dev_open,
        .mdo_dev_stop = vlan_macsec_dev_stop,
        /* SecY */
        .mdo_add_secy = vlan_macsec_add_secy,
        .mdo_upd_secy = vlan_macsec_upd_secy,
        .mdo_del_secy = vlan_macsec_del_secy,
        /* Security channels */
        .mdo_add_rxsc = vlan_macsec_add_rxsc,
        .mdo_upd_rxsc = vlan_macsec_upd_rxsc,
        .mdo_del_rxsc = vlan_macsec_del_rxsc,
        /* Security associations */
        .mdo_add_rxsa = vlan_macsec_add_rxsa,
        .mdo_upd_rxsa = vlan_macsec_upd_rxsa,
        .mdo_del_rxsa = vlan_macsec_del_rxsa,
        .mdo_add_txsa = vlan_macsec_add_txsa,
        .mdo_upd_txsa = vlan_macsec_upd_txsa,
        .mdo_del_txsa = vlan_macsec_del_txsa,
        /* Statistics */
        .mdo_get_dev_stats = vlan_macsec_get_dev_stats,
        .mdo_get_tx_sc_stats = vlan_macsec_get_tx_sc_stats,
        .mdo_get_tx_sa_stats = vlan_macsec_get_tx_sa_stats,
        .mdo_get_rx_sc_stats = vlan_macsec_get_rx_sc_stats,
        .mdo_get_rx_sa_stats = vlan_macsec_get_rx_sa_stats,
};

#endif

static const struct ethtool_ops vlan_ethtool_ops = {
        .get_link_ksettings        = vlan_ethtool_get_link_ksettings,
        .get_drvinfo                = vlan_ethtool_get_drvinfo,
        .get_link                = ethtool_op_get_link,
        .get_ts_info                = vlan_ethtool_get_ts_info,
};

static const struct net_device_ops vlan_netdev_ops = {
        .ndo_change_mtu                = vlan_dev_change_mtu,
        .ndo_init                = vlan_dev_init,
        .ndo_uninit                = vlan_dev_uninit,
        .ndo_open                = vlan_dev_open,
        .ndo_stop                = vlan_dev_stop,
        .ndo_start_xmit =  vlan_dev_hard_start_xmit,
        .ndo_validate_addr        = eth_validate_addr,
        .ndo_set_mac_address        = vlan_dev_set_mac_address,
        .ndo_set_rx_mode        = vlan_dev_set_rx_mode,
        .ndo_change_rx_flags        = vlan_dev_change_rx_flags,
        .ndo_eth_ioctl                = vlan_dev_ioctl,
        .ndo_neigh_setup        = vlan_dev_neigh_setup,
        .ndo_get_stats64        = vlan_dev_get_stats64,
#if IS_ENABLED(CONFIG_FCOE)
        .ndo_fcoe_ddp_setup        = vlan_dev_fcoe_ddp_setup,
        .ndo_fcoe_ddp_done        = vlan_dev_fcoe_ddp_done,
        .ndo_fcoe_enable        = vlan_dev_fcoe_enable,
        .ndo_fcoe_disable        = vlan_dev_fcoe_disable,
        .ndo_fcoe_ddp_target        = vlan_dev_fcoe_ddp_target,
#endif
#ifdef NETDEV_FCOE_WWNN
        .ndo_fcoe_get_wwn        = vlan_dev_fcoe_get_wwn,
#endif
#ifdef CONFIG_NET_POLL_CONTROLLER
        .ndo_poll_controller        = vlan_dev_poll_controller,
        .ndo_netpoll_setup        = vlan_dev_netpoll_setup,
        .ndo_netpoll_cleanup        = vlan_dev_netpoll_cleanup,
#endif
        .ndo_fix_features        = vlan_dev_fix_features,
        .ndo_get_iflink                = vlan_dev_get_iflink,
        .ndo_fill_forward_path        = vlan_dev_fill_forward_path,
        .ndo_hwtstamp_get        = vlan_hwtstamp_get,
        .ndo_hwtstamp_set        = vlan_hwtstamp_set,
};

static void vlan_dev_free(struct net_device *dev)
{
        struct vlan_dev_priv *vlan = vlan_dev_priv(dev);

        free_percpu(vlan->vlan_pcpu_stats);
        vlan->vlan_pcpu_stats = NULL;

        /* Get rid of the vlan's reference to real_dev */
        netdev_put(vlan->real_dev, &vlan->dev_tracker);
}

void vlan_setup(struct net_device *dev)
{
        ether_setup(dev);

        dev->priv_flags                |= IFF_802_1Q_VLAN | IFF_NO_QUEUE;
        dev->priv_flags                |= IFF_UNICAST_FLT;
        dev->priv_flags                &= ~IFF_TX_SKB_SHARING;
        netif_keep_dst(dev);

        dev->netdev_ops                = &vlan_netdev_ops;
        dev->needs_free_netdev        = true;
        dev->priv_destructor        = vlan_dev_free;
        dev->ethtool_ops        = &vlan_ethtool_ops;

#if IS_ENABLED(CONFIG_MACSEC)
        dev->macsec_ops                = &macsec_offload_ops;
#endif
        dev->min_mtu                = 0;
        dev->max_mtu                = ETH_MAX_MTU;

        eth_zero_addr(dev->broadcast);
}













































    8 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_NETDEV_RX_QUEUE_H
#define _LINUX_NETDEV_RX_QUEUE_H

#include <linux/kobject.h>
#include <linux/netdevice.h>
#include <linux/sysfs.h>
#include <net/xdp.h>
#include <net/page_pool/types.h>

/* This structure contains an instance of an RX queue. */
struct netdev_rx_queue {
        struct xdp_rxq_info                xdp_rxq;
#ifdef CONFIG_RPS
        struct rps_map __rcu                *rps_map;
        struct rps_dev_flow_table __rcu        *rps_flow_table;
#endif
        struct kobject                        kobj;
        struct net_device                *dev;
        netdevice_tracker                dev_tracker;

#ifdef CONFIG_XDP_SOCKETS
        struct xsk_buff_pool            *pool;
#endif
        /* NAPI instance for the queue
         * Readers and writers must hold RTNL
         */
        struct napi_struct                *napi;
        struct pp_memory_provider_params mp_params;
} ____cacheline_aligned_in_smp;

/*
 * RX queue sysfs structures and functions.
 */
struct rx_queue_attribute {
        struct attribute attr;
        ssize_t (*show)(struct netdev_rx_queue *queue, char *buf);
        ssize_t (*store)(struct netdev_rx_queue *queue,
                         const char *buf, size_t len);
};

static inline struct netdev_rx_queue *
__netif_get_rx_queue(struct net_device *dev, unsigned int rxq)
{
        return dev->_rx + rxq;
}

static inline unsigned int
get_netdev_rx_queue_index(struct netdev_rx_queue *queue)
{
        struct net_device *dev = queue->dev;
        int index = queue - dev->_rx;

        BUG_ON(index >= dev->num_rx_queues);
        return index;
}

int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq);

#endif



























































   35 

   36 










   35 









   35 





   36 

   36 


    1 

   36 


   36 





   36 


   23 






   19 








   65 





   81 







   21 


   22 

   22 

   21 





   18 


   18 

   19 


   19 
   19 

   16 
   19 








   52 





   49 



   39 




   13 
   13 

   52 







   11 









   59 


   59 





   58 







   61 








    3 

    4 
    3 
    3 





   68 
   65 





    6 




    3 








   62 
   68 





   37 


   37 
    5 

    5 























    2 

    2 

    2 















    2 
    2 










    2 


    2 


    2 






   22 


   19 

    3 





   26 


   23 

    3 





    9 


    6 

    3 





   53 


   53 

    2 


    2 

















    2 




    2 


    2 



    2 
























    2 


    2 




    2 


    2 




    5 


    2 
    3 





    5 


    5 
    3 

    3 





    1 







    4 
   63 




   62 

    4 

    4 
    4 

    4 




   63 


    4 
   59 











    1 















    1 
    1 


































   23 





   10 








    8 






    8 





    8 
    8 
    8 


    8 


































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Asynchronous Cryptographic Hash operations.
 *
 * This is the implementation of the ahash (asynchronous hash) API.  It differs
 * from shash (synchronous hash) in that ahash supports asynchronous operations,
 * and it hashes data from scatterlists instead of virtually addressed buffers.
 *
 * The ahash API provides access to both ahash and shash algorithms.  The shash
 * API only provides access to shash algorithms.
 *
 * Copyright (c) 2008 Loc Ho <lho@amcc.com>
 */

#include <crypto/scatterwalk.h>
#include <linux/cryptouser.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/string.h>
#include <net/netlink.h>

#include "hash.h"

#define CRYPTO_ALG_TYPE_AHASH_MASK        0x0000000e

struct crypto_hash_walk {
        char *data;

        unsigned int offset;
        unsigned int flags;

        struct page *pg;
        unsigned int entrylen;

        unsigned int total;
        struct scatterlist *sg;
};

static int hash_walk_next(struct crypto_hash_walk *walk)
{
        unsigned int offset = walk->offset;
        unsigned int nbytes = min(walk->entrylen,
                                  ((unsigned int)(PAGE_SIZE)) - offset);

        walk->data = kmap_local_page(walk->pg);
        walk->data += offset;
        walk->entrylen -= nbytes;
        return nbytes;
}

static int hash_walk_new_entry(struct crypto_hash_walk *walk)
{
        struct scatterlist *sg;

        sg = walk->sg;
        walk->offset = sg->offset;
        walk->pg = sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT);
        walk->offset = offset_in_page(walk->offset);
        walk->entrylen = sg->length;

        if (walk->entrylen > walk->total)
                walk->entrylen = walk->total;
        walk->total -= walk->entrylen;

        return hash_walk_next(walk);
}

static int crypto_hash_walk_first(struct ahash_request *req,
                                  struct crypto_hash_walk *walk)
{
        walk->total = req->nbytes;

        if (!walk->total) {
                walk->entrylen = 0;
                return 0;
        }

        walk->sg = req->src;
        walk->flags = req->base.flags;

        return hash_walk_new_entry(walk);
}

static int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
{
        walk->data -= walk->offset;

        kunmap_local(walk->data);
        crypto_yield(walk->flags);

        if (err)
                return err;

        if (walk->entrylen) {
                walk->offset = 0;
                walk->pg++;
                return hash_walk_next(walk);
        }

        if (!walk->total)
                return 0;

        walk->sg = sg_next(walk->sg);

        return hash_walk_new_entry(walk);
}

static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk)
{
        return !(walk->entrylen | walk->total);
}

/*
 * For an ahash tfm that is using an shash algorithm (instead of an ahash
 * algorithm), this returns the underlying shash tfm.
 */
static inline struct crypto_shash *ahash_to_shash(struct crypto_ahash *tfm)
{
        return *(struct crypto_shash **)crypto_ahash_ctx(tfm);
}

static inline struct shash_desc *prepare_shash_desc(struct ahash_request *req,
                                                    struct crypto_ahash *tfm)
{
        struct shash_desc *desc = ahash_request_ctx(req);

        desc->tfm = ahash_to_shash(tfm);
        return desc;
}

int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc)
{
        struct crypto_hash_walk walk;
        int nbytes;

        for (nbytes = crypto_hash_walk_first(req, &walk); nbytes > 0;
             nbytes = crypto_hash_walk_done(&walk, nbytes))
                nbytes = crypto_shash_update(desc, walk.data, nbytes);

        return nbytes;
}
EXPORT_SYMBOL_GPL(shash_ahash_update);

int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc)
{
        struct crypto_hash_walk walk;
        int nbytes;

        nbytes = crypto_hash_walk_first(req, &walk);
        if (!nbytes)
                return crypto_shash_final(desc, req->result);

        do {
                nbytes = crypto_hash_walk_last(&walk) ?
                         crypto_shash_finup(desc, walk.data, nbytes,
                                            req->result) :
                         crypto_shash_update(desc, walk.data, nbytes);
                nbytes = crypto_hash_walk_done(&walk, nbytes);
        } while (nbytes > 0);

        return nbytes;
}
EXPORT_SYMBOL_GPL(shash_ahash_finup);

int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc)
{
        unsigned int nbytes = req->nbytes;
        struct scatterlist *sg;
        unsigned int offset;
        int err;

        if (nbytes &&
            (sg = req->src, offset = sg->offset,
             nbytes <= min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset))) {
                void *data;

                data = kmap_local_page(sg_page(sg));
                err = crypto_shash_digest(desc, data + offset, nbytes,
                                          req->result);
                kunmap_local(data);
        } else
                err = crypto_shash_init(desc) ?:
                      shash_ahash_finup(req, desc);

        return err;
}
EXPORT_SYMBOL_GPL(shash_ahash_digest);

static void crypto_exit_ahash_using_shash(struct crypto_tfm *tfm)
{
        struct crypto_shash **ctx = crypto_tfm_ctx(tfm);

        crypto_free_shash(*ctx);
}

static int crypto_init_ahash_using_shash(struct crypto_tfm *tfm)
{
        struct crypto_alg *calg = tfm->__crt_alg;
        struct crypto_ahash *crt = __crypto_ahash_cast(tfm);
        struct crypto_shash **ctx = crypto_tfm_ctx(tfm);
        struct crypto_shash *shash;

        if (!crypto_mod_get(calg))
                return -EAGAIN;

        shash = crypto_create_tfm(calg, &crypto_shash_type);
        if (IS_ERR(shash)) {
                crypto_mod_put(calg);
                return PTR_ERR(shash);
        }

        crt->using_shash = true;
        *ctx = shash;
        tfm->exit = crypto_exit_ahash_using_shash;

        crypto_ahash_set_flags(crt, crypto_shash_get_flags(shash) &
                                    CRYPTO_TFM_NEED_KEY);
        crt->reqsize = sizeof(struct shash_desc) + crypto_shash_descsize(shash);

        return 0;
}

static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key,
                          unsigned int keylen)
{
        return -ENOSYS;
}

static void ahash_set_needkey(struct crypto_ahash *tfm, struct ahash_alg *alg)
{
        if (alg->setkey != ahash_nosetkey &&
            !(alg->halg.base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY))
                crypto_ahash_set_flags(tfm, CRYPTO_TFM_NEED_KEY);
}

int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
                        unsigned int keylen)
{
        if (likely(tfm->using_shash)) {
                struct crypto_shash *shash = ahash_to_shash(tfm);
                int err;

                err = crypto_shash_setkey(shash, key, keylen);
                if (unlikely(err)) {
                        crypto_ahash_set_flags(tfm,
                                               crypto_shash_get_flags(shash) &
                                               CRYPTO_TFM_NEED_KEY);
                        return err;
                }
        } else {
                struct ahash_alg *alg = crypto_ahash_alg(tfm);
                int err;

                err = alg->setkey(tfm, key, keylen);
                if (unlikely(err)) {
                        ahash_set_needkey(tfm, alg);
                        return err;
                }
        }
        crypto_ahash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
        return 0;
}
EXPORT_SYMBOL_GPL(crypto_ahash_setkey);

int crypto_ahash_init(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return crypto_shash_init(prepare_shash_desc(req, tfm));
        if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
                return -ENOKEY;
        return crypto_ahash_alg(tfm)->init(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_init);

static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt,
                          bool has_state)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        unsigned int ds = crypto_ahash_digestsize(tfm);
        struct ahash_request *subreq;
        unsigned int subreq_size;
        unsigned int reqsize;
        u8 *result;
        gfp_t gfp;
        u32 flags;

        subreq_size = sizeof(*subreq);
        reqsize = crypto_ahash_reqsize(tfm);
        reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
        subreq_size += reqsize;
        subreq_size += ds;

        flags = ahash_request_flags(req);
        gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?  GFP_KERNEL : GFP_ATOMIC;
        subreq = kmalloc(subreq_size, gfp);
        if (!subreq)
                return -ENOMEM;

        ahash_request_set_tfm(subreq, tfm);
        ahash_request_set_callback(subreq, flags, cplt, req);

        result = (u8 *)(subreq + 1) + reqsize;

        ahash_request_set_crypt(subreq, req->src, result, req->nbytes);

        if (has_state) {
                void *state;

                state = kmalloc(crypto_ahash_statesize(tfm), gfp);
                if (!state) {
                        kfree(subreq);
                        return -ENOMEM;
                }

                crypto_ahash_export(req, state);
                crypto_ahash_import(subreq, state);
                kfree_sensitive(state);
        }

        req->priv = subreq;

        return 0;
}

static void ahash_restore_req(struct ahash_request *req, int err)
{
        struct ahash_request *subreq = req->priv;

        if (!err)
                memcpy(req->result, subreq->result,
                       crypto_ahash_digestsize(crypto_ahash_reqtfm(req)));

        req->priv = NULL;

        kfree_sensitive(subreq);
}

int crypto_ahash_update(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return shash_ahash_update(req, ahash_request_ctx(req));

        return crypto_ahash_alg(tfm)->update(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_update);

int crypto_ahash_final(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return crypto_shash_final(ahash_request_ctx(req), req->result);

        return crypto_ahash_alg(tfm)->final(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_final);

int crypto_ahash_finup(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return shash_ahash_finup(req, ahash_request_ctx(req));

        return crypto_ahash_alg(tfm)->finup(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_finup);

int crypto_ahash_digest(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return shash_ahash_digest(req, prepare_shash_desc(req, tfm));

        if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
                return -ENOKEY;

        return crypto_ahash_alg(tfm)->digest(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_digest);

static void ahash_def_finup_done2(void *data, int err)
{
        struct ahash_request *areq = data;

        if (err == -EINPROGRESS)
                return;

        ahash_restore_req(areq, err);

        ahash_request_complete(areq, err);
}

static int ahash_def_finup_finish1(struct ahash_request *req, int err)
{
        struct ahash_request *subreq = req->priv;

        if (err)
                goto out;

        subreq->base.complete = ahash_def_finup_done2;

        err = crypto_ahash_alg(crypto_ahash_reqtfm(req))->final(subreq);
        if (err == -EINPROGRESS || err == -EBUSY)
                return err;

out:
        ahash_restore_req(req, err);
        return err;
}

static void ahash_def_finup_done1(void *data, int err)
{
        struct ahash_request *areq = data;
        struct ahash_request *subreq;

        if (err == -EINPROGRESS)
                goto out;

        subreq = areq->priv;
        subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;

        err = ahash_def_finup_finish1(areq, err);
        if (err == -EINPROGRESS || err == -EBUSY)
                return;

out:
        ahash_request_complete(areq, err);
}

static int ahash_def_finup(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        int err;

        err = ahash_save_req(req, ahash_def_finup_done1, true);
        if (err)
                return err;

        err = crypto_ahash_alg(tfm)->update(req->priv);
        if (err == -EINPROGRESS || err == -EBUSY)
                return err;

        return ahash_def_finup_finish1(req, err);
}

int crypto_ahash_export(struct ahash_request *req, void *out)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return crypto_shash_export(ahash_request_ctx(req), out);
        return crypto_ahash_alg(tfm)->export(req, out);
}
EXPORT_SYMBOL_GPL(crypto_ahash_export);

int crypto_ahash_import(struct ahash_request *req, const void *in)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return crypto_shash_import(prepare_shash_desc(req, tfm), in);
        if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
                return -ENOKEY;
        return crypto_ahash_alg(tfm)->import(req, in);
}
EXPORT_SYMBOL_GPL(crypto_ahash_import);

static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm)
{
        struct crypto_ahash *hash = __crypto_ahash_cast(tfm);
        struct ahash_alg *alg = crypto_ahash_alg(hash);

        alg->exit_tfm(hash);
}

static int crypto_ahash_init_tfm(struct crypto_tfm *tfm)
{
        struct crypto_ahash *hash = __crypto_ahash_cast(tfm);
        struct ahash_alg *alg = crypto_ahash_alg(hash);

        crypto_ahash_set_statesize(hash, alg->halg.statesize);

        if (tfm->__crt_alg->cra_type == &crypto_shash_type)
                return crypto_init_ahash_using_shash(tfm);

        ahash_set_needkey(hash, alg);

        if (alg->exit_tfm)
                tfm->exit = crypto_ahash_exit_tfm;

        return alg->init_tfm ? alg->init_tfm(hash) : 0;
}

static unsigned int crypto_ahash_extsize(struct crypto_alg *alg)
{
        if (alg->cra_type == &crypto_shash_type)
                return sizeof(struct crypto_shash *);

        return crypto_alg_extsize(alg);
}

static void crypto_ahash_free_instance(struct crypto_instance *inst)
{
        struct ahash_instance *ahash = ahash_instance(inst);

        ahash->free(ahash);
}

static int __maybe_unused crypto_ahash_report(
        struct sk_buff *skb, struct crypto_alg *alg)
{
        struct crypto_report_hash rhash;

        memset(&rhash, 0, sizeof(rhash));

        strscpy(rhash.type, "ahash", sizeof(rhash.type));

        rhash.blocksize = alg->cra_blocksize;
        rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize;

        return nla_put(skb, CRYPTOCFGA_REPORT_HASH, sizeof(rhash), &rhash);
}

static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
        __maybe_unused;
static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
{
        seq_printf(m, "type         : ahash\n");
        seq_printf(m, "async        : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ?
                                             "yes" : "no");
        seq_printf(m, "blocksize    : %u\n", alg->cra_blocksize);
        seq_printf(m, "digestsize   : %u\n",
                   __crypto_hash_alg_common(alg)->digestsize);
}

static const struct crypto_type crypto_ahash_type = {
        .extsize = crypto_ahash_extsize,
        .init_tfm = crypto_ahash_init_tfm,
        .free = crypto_ahash_free_instance,
#ifdef CONFIG_PROC_FS
        .show = crypto_ahash_show,
#endif
#if IS_ENABLED(CONFIG_CRYPTO_USER)
        .report = crypto_ahash_report,
#endif
        .maskclear = ~CRYPTO_ALG_TYPE_MASK,
        .maskset = CRYPTO_ALG_TYPE_AHASH_MASK,
        .type = CRYPTO_ALG_TYPE_AHASH,
        .tfmsize = offsetof(struct crypto_ahash, base),
};

int crypto_grab_ahash(struct crypto_ahash_spawn *spawn,
                      struct crypto_instance *inst,
                      const char *name, u32 type, u32 mask)
{
        spawn->base.frontend = &crypto_ahash_type;
        return crypto_grab_spawn(&spawn->base, inst, name, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_grab_ahash);

struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type,
                                        u32 mask)
{
        return crypto_alloc_tfm(alg_name, &crypto_ahash_type, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_alloc_ahash);

int crypto_has_ahash(const char *alg_name, u32 type, u32 mask)
{
        return crypto_type_has_alg(alg_name, &crypto_ahash_type, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_has_ahash);

static bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg)
{
        struct crypto_alg *alg = &halg->base;

        if (alg->cra_type == &crypto_shash_type)
                return crypto_shash_alg_has_setkey(__crypto_shash_alg(alg));

        return __crypto_ahash_alg(alg)->setkey != ahash_nosetkey;
}

struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *hash)
{
        struct hash_alg_common *halg = crypto_hash_alg_common(hash);
        struct crypto_tfm *tfm = crypto_ahash_tfm(hash);
        struct crypto_ahash *nhash;
        struct ahash_alg *alg;
        int err;

        if (!crypto_hash_alg_has_setkey(halg)) {
                tfm = crypto_tfm_get(tfm);
                if (IS_ERR(tfm))
                        return ERR_CAST(tfm);

                return hash;
        }

        nhash = crypto_clone_tfm(&crypto_ahash_type, tfm);

        if (IS_ERR(nhash))
                return nhash;

        nhash->reqsize = hash->reqsize;
        nhash->statesize = hash->statesize;

        if (likely(hash->using_shash)) {
                struct crypto_shash **nctx = crypto_ahash_ctx(nhash);
                struct crypto_shash *shash;

                shash = crypto_clone_shash(ahash_to_shash(hash));
                if (IS_ERR(shash)) {
                        err = PTR_ERR(shash);
                        goto out_free_nhash;
                }
                nhash->using_shash = true;
                *nctx = shash;
                return nhash;
        }

        err = -ENOSYS;
        alg = crypto_ahash_alg(hash);
        if (!alg->clone_tfm)
                goto out_free_nhash;

        err = alg->clone_tfm(nhash, hash);
        if (err)
                goto out_free_nhash;

        return nhash;

out_free_nhash:
        crypto_free_ahash(nhash);
        return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(crypto_clone_ahash);

static int ahash_prepare_alg(struct ahash_alg *alg)
{
        struct crypto_alg *base = &alg->halg.base;
        int err;

        if (alg->halg.statesize == 0)
                return -EINVAL;

        err = hash_prepare_alg(&alg->halg);
        if (err)
                return err;

        base->cra_type = &crypto_ahash_type;
        base->cra_flags |= CRYPTO_ALG_TYPE_AHASH;

        if (!alg->finup)
                alg->finup = ahash_def_finup;
        if (!alg->setkey)
                alg->setkey = ahash_nosetkey;

        return 0;
}

int crypto_register_ahash(struct ahash_alg *alg)
{
        struct crypto_alg *base = &alg->halg.base;
        int err;

        err = ahash_prepare_alg(alg);
        if (err)
                return err;

        return crypto_register_alg(base);
}
EXPORT_SYMBOL_GPL(crypto_register_ahash);

void crypto_unregister_ahash(struct ahash_alg *alg)
{
        crypto_unregister_alg(&alg->halg.base);
}
EXPORT_SYMBOL_GPL(crypto_unregister_ahash);

int crypto_register_ahashes(struct ahash_alg *algs, int count)
{
        int i, ret;

        for (i = 0; i < count; i++) {
                ret = crypto_register_ahash(&algs[i]);
                if (ret)
                        goto err;
        }

        return 0;

err:
        for (--i; i >= 0; --i)
                crypto_unregister_ahash(&algs[i]);

        return ret;
}
EXPORT_SYMBOL_GPL(crypto_register_ahashes);

void crypto_unregister_ahashes(struct ahash_alg *algs, int count)
{
        int i;

        for (i = count - 1; i >= 0; --i)
                crypto_unregister_ahash(&algs[i]);
}
EXPORT_SYMBOL_GPL(crypto_unregister_ahashes);

int ahash_register_instance(struct crypto_template *tmpl,
                            struct ahash_instance *inst)
{
        int err;

        if (WARN_ON(!inst->free))
                return -EINVAL;

        err = ahash_prepare_alg(&inst->alg);
        if (err)
                return err;

        return crypto_register_instance(tmpl, ahash_crypto_instance(inst));
}
EXPORT_SYMBOL_GPL(ahash_register_instance);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Asynchronous cryptographic hash type");












































































































































































































































































































































































    1 





























    1 
























    1 
























































































    1 


















































































































































    1 





































































































































































































































    1 







    1 








































































    1 


    1 













    1 









    1 









    1 






































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2006
 * NTT (Nippon Telegraph and Telephone Corporation).
 */

/*
 * Algorithm Specification
 *  https://info.isl.ntt.co.jp/crypt/eng/camellia/specifications.html
 */

#include <crypto/algapi.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/bitops.h>
#include <linux/unaligned.h>

static const u32 camellia_sp1110[256] = {
        0x70707000, 0x82828200, 0x2c2c2c00, 0xececec00,
        0xb3b3b300, 0x27272700, 0xc0c0c000, 0xe5e5e500,
        0xe4e4e400, 0x85858500, 0x57575700, 0x35353500,
        0xeaeaea00, 0x0c0c0c00, 0xaeaeae00, 0x41414100,
        0x23232300, 0xefefef00, 0x6b6b6b00, 0x93939300,
        0x45454500, 0x19191900, 0xa5a5a500, 0x21212100,
        0xededed00, 0x0e0e0e00, 0x4f4f4f00, 0x4e4e4e00,
        0x1d1d1d00, 0x65656500, 0x92929200, 0xbdbdbd00,
        0x86868600, 0xb8b8b800, 0xafafaf00, 0x8f8f8f00,
        0x7c7c7c00, 0xebebeb00, 0x1f1f1f00, 0xcecece00,
        0x3e3e3e00, 0x30303000, 0xdcdcdc00, 0x5f5f5f00,
        0x5e5e5e00, 0xc5c5c500, 0x0b0b0b00, 0x1a1a1a00,
        0xa6a6a600, 0xe1e1e100, 0x39393900, 0xcacaca00,
        0xd5d5d500, 0x47474700, 0x5d5d5d00, 0x3d3d3d00,
        0xd9d9d900, 0x01010100, 0x5a5a5a00, 0xd6d6d600,
        0x51515100, 0x56565600, 0x6c6c6c00, 0x4d4d4d00,
        0x8b8b8b00, 0x0d0d0d00, 0x9a9a9a00, 0x66666600,
        0xfbfbfb00, 0xcccccc00, 0xb0b0b000, 0x2d2d2d00,
        0x74747400, 0x12121200, 0x2b2b2b00, 0x20202000,
        0xf0f0f000, 0xb1b1b100, 0x84848400, 0x99999900,
        0xdfdfdf00, 0x4c4c4c00, 0xcbcbcb00, 0xc2c2c200,
        0x34343400, 0x7e7e7e00, 0x76767600, 0x05050500,
        0x6d6d6d00, 0xb7b7b700, 0xa9a9a900, 0x31313100,
        0xd1d1d100, 0x17171700, 0x04040400, 0xd7d7d700,
        0x14141400, 0x58585800, 0x3a3a3a00, 0x61616100,
        0xdedede00, 0x1b1b1b00, 0x11111100, 0x1c1c1c00,
        0x32323200, 0x0f0f0f00, 0x9c9c9c00, 0x16161600,
        0x53535300, 0x18181800, 0xf2f2f200, 0x22222200,
        0xfefefe00, 0x44444400, 0xcfcfcf00, 0xb2b2b200,
        0xc3c3c300, 0xb5b5b500, 0x7a7a7a00, 0x91919100,
        0x24242400, 0x08080800, 0xe8e8e800, 0xa8a8a800,
        0x60606000, 0xfcfcfc00, 0x69696900, 0x50505000,
        0xaaaaaa00, 0xd0d0d000, 0xa0a0a000, 0x7d7d7d00,
        0xa1a1a100, 0x89898900, 0x62626200, 0x97979700,
        0x54545400, 0x5b5b5b00, 0x1e1e1e00, 0x95959500,
        0xe0e0e000, 0xffffff00, 0x64646400, 0xd2d2d200,
        0x10101000, 0xc4c4c400, 0x00000000, 0x48484800,
        0xa3a3a300, 0xf7f7f700, 0x75757500, 0xdbdbdb00,
        0x8a8a8a00, 0x03030300, 0xe6e6e600, 0xdadada00,
        0x09090900, 0x3f3f3f00, 0xdddddd00, 0x94949400,
        0x87878700, 0x5c5c5c00, 0x83838300, 0x02020200,
        0xcdcdcd00, 0x4a4a4a00, 0x90909000, 0x33333300,
        0x73737300, 0x67676700, 0xf6f6f600, 0xf3f3f300,
        0x9d9d9d00, 0x7f7f7f00, 0xbfbfbf00, 0xe2e2e200,
        0x52525200, 0x9b9b9b00, 0xd8d8d800, 0x26262600,
        0xc8c8c800, 0x37373700, 0xc6c6c600, 0x3b3b3b00,
        0x81818100, 0x96969600, 0x6f6f6f00, 0x4b4b4b00,
        0x13131300, 0xbebebe00, 0x63636300, 0x2e2e2e00,
        0xe9e9e900, 0x79797900, 0xa7a7a700, 0x8c8c8c00,
        0x9f9f9f00, 0x6e6e6e00, 0xbcbcbc00, 0x8e8e8e00,
        0x29292900, 0xf5f5f500, 0xf9f9f900, 0xb6b6b600,
        0x2f2f2f00, 0xfdfdfd00, 0xb4b4b400, 0x59595900,
        0x78787800, 0x98989800, 0x06060600, 0x6a6a6a00,
        0xe7e7e700, 0x46464600, 0x71717100, 0xbababa00,
        0xd4d4d400, 0x25252500, 0xababab00, 0x42424200,
        0x88888800, 0xa2a2a200, 0x8d8d8d00, 0xfafafa00,
        0x72727200, 0x07070700, 0xb9b9b900, 0x55555500,
        0xf8f8f800, 0xeeeeee00, 0xacacac00, 0x0a0a0a00,
        0x36363600, 0x49494900, 0x2a2a2a00, 0x68686800,
        0x3c3c3c00, 0x38383800, 0xf1f1f100, 0xa4a4a400,
        0x40404000, 0x28282800, 0xd3d3d300, 0x7b7b7b00,
        0xbbbbbb00, 0xc9c9c900, 0x43434300, 0xc1c1c100,
        0x15151500, 0xe3e3e300, 0xadadad00, 0xf4f4f400,
        0x77777700, 0xc7c7c700, 0x80808000, 0x9e9e9e00,
};

static const u32 camellia_sp0222[256] = {
        0x00e0e0e0, 0x00050505, 0x00585858, 0x00d9d9d9,
        0x00676767, 0x004e4e4e, 0x00818181, 0x00cbcbcb,
        0x00c9c9c9, 0x000b0b0b, 0x00aeaeae, 0x006a6a6a,
        0x00d5d5d5, 0x00181818, 0x005d5d5d, 0x00828282,
        0x00464646, 0x00dfdfdf, 0x00d6d6d6, 0x00272727,
        0x008a8a8a, 0x00323232, 0x004b4b4b, 0x00424242,
        0x00dbdbdb, 0x001c1c1c, 0x009e9e9e, 0x009c9c9c,
        0x003a3a3a, 0x00cacaca, 0x00252525, 0x007b7b7b,
        0x000d0d0d, 0x00717171, 0x005f5f5f, 0x001f1f1f,
        0x00f8f8f8, 0x00d7d7d7, 0x003e3e3e, 0x009d9d9d,
        0x007c7c7c, 0x00606060, 0x00b9b9b9, 0x00bebebe,
        0x00bcbcbc, 0x008b8b8b, 0x00161616, 0x00343434,
        0x004d4d4d, 0x00c3c3c3, 0x00727272, 0x00959595,
        0x00ababab, 0x008e8e8e, 0x00bababa, 0x007a7a7a,
        0x00b3b3b3, 0x00020202, 0x00b4b4b4, 0x00adadad,
        0x00a2a2a2, 0x00acacac, 0x00d8d8d8, 0x009a9a9a,
        0x00171717, 0x001a1a1a, 0x00353535, 0x00cccccc,
        0x00f7f7f7, 0x00999999, 0x00616161, 0x005a5a5a,
        0x00e8e8e8, 0x00242424, 0x00565656, 0x00404040,
        0x00e1e1e1, 0x00636363, 0x00090909, 0x00333333,
        0x00bfbfbf, 0x00989898, 0x00979797, 0x00858585,
        0x00686868, 0x00fcfcfc, 0x00ececec, 0x000a0a0a,
        0x00dadada, 0x006f6f6f, 0x00535353, 0x00626262,
        0x00a3a3a3, 0x002e2e2e, 0x00080808, 0x00afafaf,
        0x00282828, 0x00b0b0b0, 0x00747474, 0x00c2c2c2,
        0x00bdbdbd, 0x00363636, 0x00222222, 0x00383838,
        0x00646464, 0x001e1e1e, 0x00393939, 0x002c2c2c,
        0x00a6a6a6, 0x00303030, 0x00e5e5e5, 0x00444444,
        0x00fdfdfd, 0x00888888, 0x009f9f9f, 0x00656565,
        0x00878787, 0x006b6b6b, 0x00f4f4f4, 0x00232323,
        0x00484848, 0x00101010, 0x00d1d1d1, 0x00515151,
        0x00c0c0c0, 0x00f9f9f9, 0x00d2d2d2, 0x00a0a0a0,
        0x00555555, 0x00a1a1a1, 0x00414141, 0x00fafafa,
        0x00434343, 0x00131313, 0x00c4c4c4, 0x002f2f2f,
        0x00a8a8a8, 0x00b6b6b6, 0x003c3c3c, 0x002b2b2b,
        0x00c1c1c1, 0x00ffffff, 0x00c8c8c8, 0x00a5a5a5,
        0x00202020, 0x00898989, 0x00000000, 0x00909090,
        0x00474747, 0x00efefef, 0x00eaeaea, 0x00b7b7b7,
        0x00151515, 0x00060606, 0x00cdcdcd, 0x00b5b5b5,
        0x00121212, 0x007e7e7e, 0x00bbbbbb, 0x00292929,
        0x000f0f0f, 0x00b8b8b8, 0x00070707, 0x00040404,
        0x009b9b9b, 0x00949494, 0x00212121, 0x00666666,
        0x00e6e6e6, 0x00cecece, 0x00ededed, 0x00e7e7e7,
        0x003b3b3b, 0x00fefefe, 0x007f7f7f, 0x00c5c5c5,
        0x00a4a4a4, 0x00373737, 0x00b1b1b1, 0x004c4c4c,
        0x00919191, 0x006e6e6e, 0x008d8d8d, 0x00767676,
        0x00030303, 0x002d2d2d, 0x00dedede, 0x00969696,
        0x00262626, 0x007d7d7d, 0x00c6c6c6, 0x005c5c5c,
        0x00d3d3d3, 0x00f2f2f2, 0x004f4f4f, 0x00191919,
        0x003f3f3f, 0x00dcdcdc, 0x00797979, 0x001d1d1d,
        0x00525252, 0x00ebebeb, 0x00f3f3f3, 0x006d6d6d,
        0x005e5e5e, 0x00fbfbfb, 0x00696969, 0x00b2b2b2,
        0x00f0f0f0, 0x00313131, 0x000c0c0c, 0x00d4d4d4,
        0x00cfcfcf, 0x008c8c8c, 0x00e2e2e2, 0x00757575,
        0x00a9a9a9, 0x004a4a4a, 0x00575757, 0x00848484,
        0x00111111, 0x00454545, 0x001b1b1b, 0x00f5f5f5,
        0x00e4e4e4, 0x000e0e0e, 0x00737373, 0x00aaaaaa,
        0x00f1f1f1, 0x00dddddd, 0x00595959, 0x00141414,
        0x006c6c6c, 0x00929292, 0x00545454, 0x00d0d0d0,
        0x00787878, 0x00707070, 0x00e3e3e3, 0x00494949,
        0x00808080, 0x00505050, 0x00a7a7a7, 0x00f6f6f6,
        0x00777777, 0x00939393, 0x00868686, 0x00838383,
        0x002a2a2a, 0x00c7c7c7, 0x005b5b5b, 0x00e9e9e9,
        0x00eeeeee, 0x008f8f8f, 0x00010101, 0x003d3d3d,
};

static const u32 camellia_sp3033[256] = {
        0x38003838, 0x41004141, 0x16001616, 0x76007676,
        0xd900d9d9, 0x93009393, 0x60006060, 0xf200f2f2,
        0x72007272, 0xc200c2c2, 0xab00abab, 0x9a009a9a,
        0x75007575, 0x06000606, 0x57005757, 0xa000a0a0,
        0x91009191, 0xf700f7f7, 0xb500b5b5, 0xc900c9c9,
        0xa200a2a2, 0x8c008c8c, 0xd200d2d2, 0x90009090,
        0xf600f6f6, 0x07000707, 0xa700a7a7, 0x27002727,
        0x8e008e8e, 0xb200b2b2, 0x49004949, 0xde00dede,
        0x43004343, 0x5c005c5c, 0xd700d7d7, 0xc700c7c7,
        0x3e003e3e, 0xf500f5f5, 0x8f008f8f, 0x67006767,
        0x1f001f1f, 0x18001818, 0x6e006e6e, 0xaf00afaf,
        0x2f002f2f, 0xe200e2e2, 0x85008585, 0x0d000d0d,
        0x53005353, 0xf000f0f0, 0x9c009c9c, 0x65006565,
        0xea00eaea, 0xa300a3a3, 0xae00aeae, 0x9e009e9e,
        0xec00ecec, 0x80008080, 0x2d002d2d, 0x6b006b6b,
        0xa800a8a8, 0x2b002b2b, 0x36003636, 0xa600a6a6,
        0xc500c5c5, 0x86008686, 0x4d004d4d, 0x33003333,
        0xfd00fdfd, 0x66006666, 0x58005858, 0x96009696,
        0x3a003a3a, 0x09000909, 0x95009595, 0x10001010,
        0x78007878, 0xd800d8d8, 0x42004242, 0xcc00cccc,
        0xef00efef, 0x26002626, 0xe500e5e5, 0x61006161,
        0x1a001a1a, 0x3f003f3f, 0x3b003b3b, 0x82008282,
        0xb600b6b6, 0xdb00dbdb, 0xd400d4d4, 0x98009898,
        0xe800e8e8, 0x8b008b8b, 0x02000202, 0xeb00ebeb,
        0x0a000a0a, 0x2c002c2c, 0x1d001d1d, 0xb000b0b0,
        0x6f006f6f, 0x8d008d8d, 0x88008888, 0x0e000e0e,
        0x19001919, 0x87008787, 0x4e004e4e, 0x0b000b0b,
        0xa900a9a9, 0x0c000c0c, 0x79007979, 0x11001111,
        0x7f007f7f, 0x22002222, 0xe700e7e7, 0x59005959,
        0xe100e1e1, 0xda00dada, 0x3d003d3d, 0xc800c8c8,
        0x12001212, 0x04000404, 0x74007474, 0x54005454,
        0x30003030, 0x7e007e7e, 0xb400b4b4, 0x28002828,
        0x55005555, 0x68006868, 0x50005050, 0xbe00bebe,
        0xd000d0d0, 0xc400c4c4, 0x31003131, 0xcb00cbcb,
        0x2a002a2a, 0xad00adad, 0x0f000f0f, 0xca00caca,
        0x70007070, 0xff00ffff, 0x32003232, 0x69006969,
        0x08000808, 0x62006262, 0x00000000, 0x24002424,
        0xd100d1d1, 0xfb00fbfb, 0xba00baba, 0xed00eded,
        0x45004545, 0x81008181, 0x73007373, 0x6d006d6d,
        0x84008484, 0x9f009f9f, 0xee00eeee, 0x4a004a4a,
        0xc300c3c3, 0x2e002e2e, 0xc100c1c1, 0x01000101,
        0xe600e6e6, 0x25002525, 0x48004848, 0x99009999,
        0xb900b9b9, 0xb300b3b3, 0x7b007b7b, 0xf900f9f9,
        0xce00cece, 0xbf00bfbf, 0xdf00dfdf, 0x71007171,
        0x29002929, 0xcd00cdcd, 0x6c006c6c, 0x13001313,
        0x64006464, 0x9b009b9b, 0x63006363, 0x9d009d9d,
        0xc000c0c0, 0x4b004b4b, 0xb700b7b7, 0xa500a5a5,
        0x89008989, 0x5f005f5f, 0xb100b1b1, 0x17001717,
        0xf400f4f4, 0xbc00bcbc, 0xd300d3d3, 0x46004646,
        0xcf00cfcf, 0x37003737, 0x5e005e5e, 0x47004747,
        0x94009494, 0xfa00fafa, 0xfc00fcfc, 0x5b005b5b,
        0x97009797, 0xfe00fefe, 0x5a005a5a, 0xac00acac,
        0x3c003c3c, 0x4c004c4c, 0x03000303, 0x35003535,
        0xf300f3f3, 0x23002323, 0xb800b8b8, 0x5d005d5d,
        0x6a006a6a, 0x92009292, 0xd500d5d5, 0x21002121,
        0x44004444, 0x51005151, 0xc600c6c6, 0x7d007d7d,
        0x39003939, 0x83008383, 0xdc00dcdc, 0xaa00aaaa,
        0x7c007c7c, 0x77007777, 0x56005656, 0x05000505,
        0x1b001b1b, 0xa400a4a4, 0x15001515, 0x34003434,
        0x1e001e1e, 0x1c001c1c, 0xf800f8f8, 0x52005252,
        0x20002020, 0x14001414, 0xe900e9e9, 0xbd00bdbd,
        0xdd00dddd, 0xe400e4e4, 0xa100a1a1, 0xe000e0e0,
        0x8a008a8a, 0xf100f1f1, 0xd600d6d6, 0x7a007a7a,
        0xbb00bbbb, 0xe300e3e3, 0x40004040, 0x4f004f4f,
};

static const u32 camellia_sp4404[256] = {
        0x70700070, 0x2c2c002c, 0xb3b300b3, 0xc0c000c0,
        0xe4e400e4, 0x57570057, 0xeaea00ea, 0xaeae00ae,
        0x23230023, 0x6b6b006b, 0x45450045, 0xa5a500a5,
        0xeded00ed, 0x4f4f004f, 0x1d1d001d, 0x92920092,
        0x86860086, 0xafaf00af, 0x7c7c007c, 0x1f1f001f,
        0x3e3e003e, 0xdcdc00dc, 0x5e5e005e, 0x0b0b000b,
        0xa6a600a6, 0x39390039, 0xd5d500d5, 0x5d5d005d,
        0xd9d900d9, 0x5a5a005a, 0x51510051, 0x6c6c006c,
        0x8b8b008b, 0x9a9a009a, 0xfbfb00fb, 0xb0b000b0,
        0x74740074, 0x2b2b002b, 0xf0f000f0, 0x84840084,
        0xdfdf00df, 0xcbcb00cb, 0x34340034, 0x76760076,
        0x6d6d006d, 0xa9a900a9, 0xd1d100d1, 0x04040004,
        0x14140014, 0x3a3a003a, 0xdede00de, 0x11110011,
        0x32320032, 0x9c9c009c, 0x53530053, 0xf2f200f2,
        0xfefe00fe, 0xcfcf00cf, 0xc3c300c3, 0x7a7a007a,
        0x24240024, 0xe8e800e8, 0x60600060, 0x69690069,
        0xaaaa00aa, 0xa0a000a0, 0xa1a100a1, 0x62620062,
        0x54540054, 0x1e1e001e, 0xe0e000e0, 0x64640064,
        0x10100010, 0x00000000, 0xa3a300a3, 0x75750075,
        0x8a8a008a, 0xe6e600e6, 0x09090009, 0xdddd00dd,
        0x87870087, 0x83830083, 0xcdcd00cd, 0x90900090,
        0x73730073, 0xf6f600f6, 0x9d9d009d, 0xbfbf00bf,
        0x52520052, 0xd8d800d8, 0xc8c800c8, 0xc6c600c6,
        0x81810081, 0x6f6f006f, 0x13130013, 0x63630063,
        0xe9e900e9, 0xa7a700a7, 0x9f9f009f, 0xbcbc00bc,
        0x29290029, 0xf9f900f9, 0x2f2f002f, 0xb4b400b4,
        0x78780078, 0x06060006, 0xe7e700e7, 0x71710071,
        0xd4d400d4, 0xabab00ab, 0x88880088, 0x8d8d008d,
        0x72720072, 0xb9b900b9, 0xf8f800f8, 0xacac00ac,
        0x36360036, 0x2a2a002a, 0x3c3c003c, 0xf1f100f1,
        0x40400040, 0xd3d300d3, 0xbbbb00bb, 0x43430043,
        0x15150015, 0xadad00ad, 0x77770077, 0x80800080,
        0x82820082, 0xecec00ec, 0x27270027, 0xe5e500e5,
        0x85850085, 0x35350035, 0x0c0c000c, 0x41410041,
        0xefef00ef, 0x93930093, 0x19190019, 0x21210021,
        0x0e0e000e, 0x4e4e004e, 0x65650065, 0xbdbd00bd,
        0xb8b800b8, 0x8f8f008f, 0xebeb00eb, 0xcece00ce,
        0x30300030, 0x5f5f005f, 0xc5c500c5, 0x1a1a001a,
        0xe1e100e1, 0xcaca00ca, 0x47470047, 0x3d3d003d,
        0x01010001, 0xd6d600d6, 0x56560056, 0x4d4d004d,
        0x0d0d000d, 0x66660066, 0xcccc00cc, 0x2d2d002d,
        0x12120012, 0x20200020, 0xb1b100b1, 0x99990099,
        0x4c4c004c, 0xc2c200c2, 0x7e7e007e, 0x05050005,
        0xb7b700b7, 0x31310031, 0x17170017, 0xd7d700d7,
        0x58580058, 0x61610061, 0x1b1b001b, 0x1c1c001c,
        0x0f0f000f, 0x16160016, 0x18180018, 0x22220022,
        0x44440044, 0xb2b200b2, 0xb5b500b5, 0x91910091,
        0x08080008, 0xa8a800a8, 0xfcfc00fc, 0x50500050,
        0xd0d000d0, 0x7d7d007d, 0x89890089, 0x97970097,
        0x5b5b005b, 0x95950095, 0xffff00ff, 0xd2d200d2,
        0xc4c400c4, 0x48480048, 0xf7f700f7, 0xdbdb00db,
        0x03030003, 0xdada00da, 0x3f3f003f, 0x94940094,
        0x5c5c005c, 0x02020002, 0x4a4a004a, 0x33330033,
        0x67670067, 0xf3f300f3, 0x7f7f007f, 0xe2e200e2,
        0x9b9b009b, 0x26260026, 0x37370037, 0x3b3b003b,
        0x96960096, 0x4b4b004b, 0xbebe00be, 0x2e2e002e,
        0x79790079, 0x8c8c008c, 0x6e6e006e, 0x8e8e008e,
        0xf5f500f5, 0xb6b600b6, 0xfdfd00fd, 0x59590059,
        0x98980098, 0x6a6a006a, 0x46460046, 0xbaba00ba,
        0x25250025, 0x42420042, 0xa2a200a2, 0xfafa00fa,
        0x07070007, 0x55550055, 0xeeee00ee, 0x0a0a000a,
        0x49490049, 0x68680068, 0x38380038, 0xa4a400a4,
        0x28280028, 0x7b7b007b, 0xc9c900c9, 0xc1c100c1,
        0xe3e300e3, 0xf4f400f4, 0xc7c700c7, 0x9e9e009e,
};


#define CAMELLIA_MIN_KEY_SIZE        16
#define CAMELLIA_MAX_KEY_SIZE        32
#define CAMELLIA_BLOCK_SIZE          16
#define CAMELLIA_TABLE_BYTE_LEN     272

/*
 * NB: L and R below stand for 'left' and 'right' as in written numbers.
 * That is, in (xxxL,xxxR) pair xxxL holds most significant digits,
 * _not_ least significant ones!
 */


/* key constants */

#define CAMELLIA_SIGMA1L (0xA09E667FL)
#define CAMELLIA_SIGMA1R (0x3BCC908BL)
#define CAMELLIA_SIGMA2L (0xB67AE858L)
#define CAMELLIA_SIGMA2R (0x4CAA73B2L)
#define CAMELLIA_SIGMA3L (0xC6EF372FL)
#define CAMELLIA_SIGMA3R (0xE94F82BEL)
#define CAMELLIA_SIGMA4L (0x54FF53A5L)
#define CAMELLIA_SIGMA4R (0xF1D36F1CL)
#define CAMELLIA_SIGMA5L (0x10E527FAL)
#define CAMELLIA_SIGMA5R (0xDE682D1DL)
#define CAMELLIA_SIGMA6L (0xB05688C2L)
#define CAMELLIA_SIGMA6R (0xB3E6C1FDL)

/*
 *  macros
 */
#define ROLDQ(ll, lr, rl, rr, w0, w1, bits) ({                \
        w0 = ll;                                        \
        ll = (ll << bits) + (lr >> (32 - bits));        \
        lr = (lr << bits) + (rl >> (32 - bits));        \
        rl = (rl << bits) + (rr >> (32 - bits));        \
        rr = (rr << bits) + (w0 >> (32 - bits));        \
})

#define ROLDQo32(ll, lr, rl, rr, w0, w1, bits) ({        \
        w0 = ll;                                        \
        w1 = lr;                                        \
        ll = (lr << (bits - 32)) + (rl >> (64 - bits));        \
        lr = (rl << (bits - 32)) + (rr >> (64 - bits));        \
        rl = (rr << (bits - 32)) + (w0 >> (64 - bits));        \
        rr = (w0 << (bits - 32)) + (w1 >> (64 - bits));        \
})

#define CAMELLIA_F(xl, xr, kl, kr, yl, yr, il, ir, t0, t1) ({        \
        il = xl ^ kl;                                                \
        ir = xr ^ kr;                                                \
        t0 = il >> 16;                                                \
        t1 = ir >> 16;                                                \
        yl = camellia_sp1110[(u8)(ir)]                                \
           ^ camellia_sp0222[(u8)(t1 >> 8)]                        \
           ^ camellia_sp3033[(u8)(t1)]                                \
           ^ camellia_sp4404[(u8)(ir >> 8)];                        \
        yr = camellia_sp1110[(u8)(t0 >> 8)]                        \
           ^ camellia_sp0222[(u8)(t0)]                                \
           ^ camellia_sp3033[(u8)(il >> 8)]                        \
           ^ camellia_sp4404[(u8)(il)];                                \
        yl ^= yr;                                                \
        yr = ror32(yr, 8);                                        \
        yr ^= yl;                                                \
})

#define SUBKEY_L(INDEX) (subkey[(INDEX)*2])
#define SUBKEY_R(INDEX) (subkey[(INDEX)*2 + 1])

static void camellia_setup_tail(u32 *subkey, u32 *subL, u32 *subR, int max)
{
        u32 dw, tl, tr;
        u32 kw4l, kw4r;

        /* absorb kw2 to other subkeys */
        /* round 2 */
        subL[3] ^= subL[1]; subR[3] ^= subR[1];
        /* round 4 */
        subL[5] ^= subL[1]; subR[5] ^= subR[1];
        /* round 6 */
        subL[7] ^= subL[1]; subR[7] ^= subR[1];
        subL[1] ^= subR[1] & ~subR[9];
        dw = subL[1] & subL[9];
        subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl2) */
        /* round 8 */
        subL[11] ^= subL[1]; subR[11] ^= subR[1];
        /* round 10 */
        subL[13] ^= subL[1]; subR[13] ^= subR[1];
        /* round 12 */
        subL[15] ^= subL[1]; subR[15] ^= subR[1];
        subL[1] ^= subR[1] & ~subR[17];
        dw = subL[1] & subL[17];
        subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl4) */
        /* round 14 */
        subL[19] ^= subL[1]; subR[19] ^= subR[1];
        /* round 16 */
        subL[21] ^= subL[1]; subR[21] ^= subR[1];
        /* round 18 */
        subL[23] ^= subL[1]; subR[23] ^= subR[1];
        if (max == 24) {
                /* kw3 */
                subL[24] ^= subL[1]; subR[24] ^= subR[1];

        /* absorb kw4 to other subkeys */
                kw4l = subL[25]; kw4r = subR[25];
        } else {
                subL[1] ^= subR[1] & ~subR[25];
                dw = subL[1] & subL[25];
                subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl6) */
                /* round 20 */
                subL[27] ^= subL[1]; subR[27] ^= subR[1];
                /* round 22 */
                subL[29] ^= subL[1]; subR[29] ^= subR[1];
                /* round 24 */
                subL[31] ^= subL[1]; subR[31] ^= subR[1];
                /* kw3 */
                subL[32] ^= subL[1]; subR[32] ^= subR[1];

        /* absorb kw4 to other subkeys */
                kw4l = subL[33]; kw4r = subR[33];
                /* round 23 */
                subL[30] ^= kw4l; subR[30] ^= kw4r;
                /* round 21 */
                subL[28] ^= kw4l; subR[28] ^= kw4r;
                /* round 19 */
                subL[26] ^= kw4l; subR[26] ^= kw4r;
                kw4l ^= kw4r & ~subR[24];
                dw = kw4l & subL[24];
                kw4r ^= rol32(dw, 1); /* modified for FL(kl5) */
        }
        /* round 17 */
        subL[22] ^= kw4l; subR[22] ^= kw4r;
        /* round 15 */
        subL[20] ^= kw4l; subR[20] ^= kw4r;
        /* round 13 */
        subL[18] ^= kw4l; subR[18] ^= kw4r;
        kw4l ^= kw4r & ~subR[16];
        dw = kw4l & subL[16];
        kw4r ^= rol32(dw, 1); /* modified for FL(kl3) */
        /* round 11 */
        subL[14] ^= kw4l; subR[14] ^= kw4r;
        /* round 9 */
        subL[12] ^= kw4l; subR[12] ^= kw4r;
        /* round 7 */
        subL[10] ^= kw4l; subR[10] ^= kw4r;
        kw4l ^= kw4r & ~subR[8];
        dw = kw4l & subL[8];
        kw4r ^= rol32(dw, 1); /* modified for FL(kl1) */
        /* round 5 */
        subL[6] ^= kw4l; subR[6] ^= kw4r;
        /* round 3 */
        subL[4] ^= kw4l; subR[4] ^= kw4r;
        /* round 1 */
        subL[2] ^= kw4l; subR[2] ^= kw4r;
        /* kw1 */
        subL[0] ^= kw4l; subR[0] ^= kw4r;

        /* key XOR is end of F-function */
        SUBKEY_L(0) = subL[0] ^ subL[2];/* kw1 */
        SUBKEY_R(0) = subR[0] ^ subR[2];
        SUBKEY_L(2) = subL[3];       /* round 1 */
        SUBKEY_R(2) = subR[3];
        SUBKEY_L(3) = subL[2] ^ subL[4]; /* round 2 */
        SUBKEY_R(3) = subR[2] ^ subR[4];
        SUBKEY_L(4) = subL[3] ^ subL[5]; /* round 3 */
        SUBKEY_R(4) = subR[3] ^ subR[5];
        SUBKEY_L(5) = subL[4] ^ subL[6]; /* round 4 */
        SUBKEY_R(5) = subR[4] ^ subR[6];
        SUBKEY_L(6) = subL[5] ^ subL[7]; /* round 5 */
        SUBKEY_R(6) = subR[5] ^ subR[7];
        tl = subL[10] ^ (subR[10] & ~subR[8]);
        dw = tl & subL[8];  /* FL(kl1) */
        tr = subR[10] ^ rol32(dw, 1);
        SUBKEY_L(7) = subL[6] ^ tl; /* round 6 */
        SUBKEY_R(7) = subR[6] ^ tr;
        SUBKEY_L(8) = subL[8];       /* FL(kl1) */
        SUBKEY_R(8) = subR[8];
        SUBKEY_L(9) = subL[9];       /* FLinv(kl2) */
        SUBKEY_R(9) = subR[9];
        tl = subL[7] ^ (subR[7] & ~subR[9]);
        dw = tl & subL[9];  /* FLinv(kl2) */
        tr = subR[7] ^ rol32(dw, 1);
        SUBKEY_L(10) = tl ^ subL[11]; /* round 7 */
        SUBKEY_R(10) = tr ^ subR[11];
        SUBKEY_L(11) = subL[10] ^ subL[12]; /* round 8 */
        SUBKEY_R(11) = subR[10] ^ subR[12];
        SUBKEY_L(12) = subL[11] ^ subL[13]; /* round 9 */
        SUBKEY_R(12) = subR[11] ^ subR[13];
        SUBKEY_L(13) = subL[12] ^ subL[14]; /* round 10 */
        SUBKEY_R(13) = subR[12] ^ subR[14];
        SUBKEY_L(14) = subL[13] ^ subL[15]; /* round 11 */
        SUBKEY_R(14) = subR[13] ^ subR[15];
        tl = subL[18] ^ (subR[18] & ~subR[16]);
        dw = tl & subL[16]; /* FL(kl3) */
        tr = subR[18] ^ rol32(dw, 1);
        SUBKEY_L(15) = subL[14] ^ tl; /* round 12 */
        SUBKEY_R(15) = subR[14] ^ tr;
        SUBKEY_L(16) = subL[16];     /* FL(kl3) */
        SUBKEY_R(16) = subR[16];
        SUBKEY_L(17) = subL[17];     /* FLinv(kl4) */
        SUBKEY_R(17) = subR[17];
        tl = subL[15] ^ (subR[15] & ~subR[17]);
        dw = tl & subL[17]; /* FLinv(kl4) */
        tr = subR[15] ^ rol32(dw, 1);
        SUBKEY_L(18) = tl ^ subL[19]; /* round 13 */
        SUBKEY_R(18) = tr ^ subR[19];
        SUBKEY_L(19) = subL[18] ^ subL[20]; /* round 14 */
        SUBKEY_R(19) = subR[18] ^ subR[20];
        SUBKEY_L(20) = subL[19] ^ subL[21]; /* round 15 */
        SUBKEY_R(20) = subR[19] ^ subR[21];
        SUBKEY_L(21) = subL[20] ^ subL[22]; /* round 16 */
        SUBKEY_R(21) = subR[20] ^ subR[22];
        SUBKEY_L(22) = subL[21] ^ subL[23]; /* round 17 */
        SUBKEY_R(22) = subR[21] ^ subR[23];
        if (max == 24) {
                SUBKEY_L(23) = subL[22];     /* round 18 */
                SUBKEY_R(23) = subR[22];
                SUBKEY_L(24) = subL[24] ^ subL[23]; /* kw3 */
                SUBKEY_R(24) = subR[24] ^ subR[23];
        } else {
                tl = subL[26] ^ (subR[26] & ~subR[24]);
                dw = tl & subL[24]; /* FL(kl5) */
                tr = subR[26] ^ rol32(dw, 1);
                SUBKEY_L(23) = subL[22] ^ tl; /* round 18 */
                SUBKEY_R(23) = subR[22] ^ tr;
                SUBKEY_L(24) = subL[24];     /* FL(kl5) */
                SUBKEY_R(24) = subR[24];
                SUBKEY_L(25) = subL[25];     /* FLinv(kl6) */
                SUBKEY_R(25) = subR[25];
                tl = subL[23] ^ (subR[23] & ~subR[25]);
                dw = tl & subL[25]; /* FLinv(kl6) */
                tr = subR[23] ^ rol32(dw, 1);
                SUBKEY_L(26) = tl ^ subL[27]; /* round 19 */
                SUBKEY_R(26) = tr ^ subR[27];
                SUBKEY_L(27) = subL[26] ^ subL[28]; /* round 20 */
                SUBKEY_R(27) = subR[26] ^ subR[28];
                SUBKEY_L(28) = subL[27] ^ subL[29]; /* round 21 */
                SUBKEY_R(28) = subR[27] ^ subR[29];
                SUBKEY_L(29) = subL[28] ^ subL[30]; /* round 22 */
                SUBKEY_R(29) = subR[28] ^ subR[30];
                SUBKEY_L(30) = subL[29] ^ subL[31]; /* round 23 */
                SUBKEY_R(30) = subR[29] ^ subR[31];
                SUBKEY_L(31) = subL[30];     /* round 24 */
                SUBKEY_R(31) = subR[30];
                SUBKEY_L(32) = subL[32] ^ subL[31]; /* kw3 */
                SUBKEY_R(32) = subR[32] ^ subR[31];
        }
}

static void camellia_setup128(const unsigned char *key, u32 *subkey)
{
        u32 kll, klr, krl, krr;
        u32 il, ir, t0, t1, w0, w1;
        u32 subL[26];
        u32 subR[26];

        /**
         *  k == kll || klr || krl || krr (|| is concatenation)
         */
        kll = get_unaligned_be32(key);
        klr = get_unaligned_be32(key + 4);
        krl = get_unaligned_be32(key + 8);
        krr = get_unaligned_be32(key + 12);

        /* generate KL dependent subkeys */
        /* kw1 */
        subL[0] = kll; subR[0] = klr;
        /* kw2 */
        subL[1] = krl; subR[1] = krr;
        /* rotation left shift 15bit */
        ROLDQ(kll, klr, krl, krr, w0, w1, 15);
        /* k3 */
        subL[4] = kll; subR[4] = klr;
        /* k4 */
        subL[5] = krl; subR[5] = krr;
        /* rotation left shift 15+30bit */
        ROLDQ(kll, klr, krl, krr, w0, w1, 30);
        /* k7 */
        subL[10] = kll; subR[10] = klr;
        /* k8 */
        subL[11] = krl; subR[11] = krr;
        /* rotation left shift 15+30+15bit */
        ROLDQ(kll, klr, krl, krr, w0, w1, 15);
        /* k10 */
        subL[13] = krl; subR[13] = krr;
        /* rotation left shift 15+30+15+17 bit */
        ROLDQ(kll, klr, krl, krr, w0, w1, 17);
        /* kl3 */
        subL[16] = kll; subR[16] = klr;
        /* kl4 */
        subL[17] = krl; subR[17] = krr;
        /* rotation left shift 15+30+15+17+17 bit */
        ROLDQ(kll, klr, krl, krr, w0, w1, 17);
        /* k13 */
        subL[18] = kll; subR[18] = klr;
        /* k14 */
        subL[19] = krl; subR[19] = krr;
        /* rotation left shift 15+30+15+17+17+17 bit */
        ROLDQ(kll, klr, krl, krr, w0, w1, 17);
        /* k17 */
        subL[22] = kll; subR[22] = klr;
        /* k18 */
        subL[23] = krl; subR[23] = krr;

        /* generate KA */
        kll = subL[0]; klr = subR[0];
        krl = subL[1]; krr = subR[1];
        CAMELLIA_F(kll, klr,
                   CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R,
                   w0, w1, il, ir, t0, t1);
        krl ^= w0; krr ^= w1;
        CAMELLIA_F(krl, krr,
                   CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R,
                   kll, klr, il, ir, t0, t1);
        /* current status == (kll, klr, w0, w1) */
        CAMELLIA_F(kll, klr,
                   CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R,
                   krl, krr, il, ir, t0, t1);
        krl ^= w0; krr ^= w1;
        CAMELLIA_F(krl, krr,
                   CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R,
                   w0, w1, il, ir, t0, t1);
        kll ^= w0; klr ^= w1;

        /* generate KA dependent subkeys */
        /* k1, k2 */
        subL[2] = kll; subR[2] = klr;
        subL[3] = krl; subR[3] = krr;
        ROLDQ(kll, klr, krl, krr, w0, w1, 15);
        /* k5,k6 */
        subL[6] = kll; subR[6] = klr;
        subL[7] = krl; subR[7] = krr;
        ROLDQ(kll, klr, krl, krr, w0, w1, 15);
        /* kl1, kl2 */
        subL[8] = kll; subR[8] = klr;
        subL[9] = krl; subR[9] = krr;
        ROLDQ(kll, klr, krl, krr, w0, w1, 15);
        /* k9 */
        subL[12] = kll; subR[12] = klr;
        ROLDQ(kll, klr, krl, krr, w0, w1, 15);
        /* k11, k12 */
        subL[14] = kll; subR[14] = klr;
        subL[15] = krl; subR[15] = krr;
        ROLDQo32(kll, klr, krl, krr, w0, w1, 34);
        /* k15, k16 */
        subL[20] = kll; subR[20] = klr;
        subL[21] = krl; subR[21] = krr;
        ROLDQ(kll, klr, krl, krr, w0, w1, 17);
        /* kw3, kw4 */
        subL[24] = kll; subR[24] = klr;
        subL[25] = krl; subR[25] = krr;

        camellia_setup_tail(subkey, subL, subR, 24);
}

static void camellia_setup256(const unsigned char *key, u32 *subkey)
{
        u32 kll, klr, krl, krr;        /* left half of key */
        u32 krll, krlr, krrl, krrr;    /* right half of key */
        u32 il, ir, t0, t1, w0, w1;    /* temporary variables */
        u32 subL[34];
        u32 subR[34];

        /**
         *  key = (kll || klr || krl || krr || krll || krlr || krrl || krrr)
         *  (|| is concatenation)
         */
        kll = get_unaligned_be32(key);
        klr = get_unaligned_be32(key + 4);
        krl = get_unaligned_be32(key + 8);
        krr = get_unaligned_be32(key + 12);
        krll = get_unaligned_be32(key + 16);
        krlr = get_unaligned_be32(key + 20);
        krrl = get_unaligned_be32(key + 24);
        krrr = get_unaligned_be32(key + 28);

        /* generate KL dependent subkeys */
        /* kw1 */
        subL[0] = kll; subR[0] = klr;
        /* kw2 */
        subL[1] = krl; subR[1] = krr;
        ROLDQo32(kll, klr, krl, krr, w0, w1, 45);
        /* k9 */
        subL[12] = kll; subR[12] = klr;
        /* k10 */
        subL[13] = krl; subR[13] = krr;
        ROLDQ(kll, klr, krl, krr, w0, w1, 15);
        /* kl3 */
        subL[16] = kll; subR[16] = klr;
        /* kl4 */
        subL[17] = krl; subR[17] = krr;
        ROLDQ(kll, klr, krl, krr, w0, w1, 17);
        /* k17 */
        subL[22] = kll; subR[22] = klr;
        /* k18 */
        subL[23] = krl; subR[23] = krr;
        ROLDQo32(kll, klr, krl, krr, w0, w1, 34);
        /* k23 */
        subL[30] = kll; subR[30] = klr;
        /* k24 */
        subL[31] = krl; subR[31] = krr;

        /* generate KR dependent subkeys */
        ROLDQ(krll, krlr, krrl, krrr, w0, w1, 15);
        /* k3 */
        subL[4] = krll; subR[4] = krlr;
        /* k4 */
        subL[5] = krrl; subR[5] = krrr;
        ROLDQ(krll, krlr, krrl, krrr, w0, w1, 15);
        /* kl1 */
        subL[8] = krll; subR[8] = krlr;
        /* kl2 */
        subL[9] = krrl; subR[9] = krrr;
        ROLDQ(krll, krlr, krrl, krrr, w0, w1, 30);
        /* k13 */
        subL[18] = krll; subR[18] = krlr;
        /* k14 */
        subL[19] = krrl; subR[19] = krrr;
        ROLDQo32(krll, krlr, krrl, krrr, w0, w1, 34);
        /* k19 */
        subL[26] = krll; subR[26] = krlr;
        /* k20 */
        subL[27] = krrl; subR[27] = krrr;
        ROLDQo32(krll, krlr, krrl, krrr, w0, w1, 34);

        /* generate KA */
        kll = subL[0] ^ krll; klr = subR[0] ^ krlr;
        krl = subL[1] ^ krrl; krr = subR[1] ^ krrr;
        CAMELLIA_F(kll, klr,
                   CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R,
                   w0, w1, il, ir, t0, t1);
        krl ^= w0; krr ^= w1;
        CAMELLIA_F(krl, krr,
                   CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R,
                   kll, klr, il, ir, t0, t1);
        kll ^= krll; klr ^= krlr;
        CAMELLIA_F(kll, klr,
                   CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R,
                   krl, krr, il, ir, t0, t1);
        krl ^= w0 ^ krrl; krr ^= w1 ^ krrr;
        CAMELLIA_F(krl, krr,
                   CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R,
                   w0, w1, il, ir, t0, t1);
        kll ^= w0; klr ^= w1;

        /* generate KB */
        krll ^= kll; krlr ^= klr;
        krrl ^= krl; krrr ^= krr;
        CAMELLIA_F(krll, krlr,
                   CAMELLIA_SIGMA5L, CAMELLIA_SIGMA5R,
                   w0, w1, il, ir, t0, t1);
        krrl ^= w0; krrr ^= w1;
        CAMELLIA_F(krrl, krrr,
                   CAMELLIA_SIGMA6L, CAMELLIA_SIGMA6R,
                   w0, w1, il, ir, t0, t1);
        krll ^= w0; krlr ^= w1;

        /* generate KA dependent subkeys */
        ROLDQ(kll, klr, krl, krr, w0, w1, 15);
        /* k5 */
        subL[6] = kll; subR[6] = klr;
        /* k6 */
        subL[7] = krl; subR[7] = krr;
        ROLDQ(kll, klr, krl, krr, w0, w1, 30);
        /* k11 */
        subL[14] = kll; subR[14] = klr;
        /* k12 */
        subL[15] = krl; subR[15] = krr;
        /* rotation left shift 32bit */
        /* kl5 */
        subL[24] = klr; subR[24] = krl;
        /* kl6 */
        subL[25] = krr; subR[25] = kll;
        /* rotation left shift 49 from k11,k12 -> k21,k22 */
        ROLDQo32(kll, klr, krl, krr, w0, w1, 49);
        /* k21 */
        subL[28] = kll; subR[28] = klr;
        /* k22 */
        subL[29] = krl; subR[29] = krr;

        /* generate KB dependent subkeys */
        /* k1 */
        subL[2] = krll; subR[2] = krlr;
        /* k2 */
        subL[3] = krrl; subR[3] = krrr;
        ROLDQ(krll, krlr, krrl, krrr, w0, w1, 30);
        /* k7 */
        subL[10] = krll; subR[10] = krlr;
        /* k8 */
        subL[11] = krrl; subR[11] = krrr;
        ROLDQ(krll, krlr, krrl, krrr, w0, w1, 30);
        /* k15 */
        subL[20] = krll; subR[20] = krlr;
        /* k16 */
        subL[21] = krrl; subR[21] = krrr;
        ROLDQo32(krll, krlr, krrl, krrr, w0, w1, 51);
        /* kw3 */
        subL[32] = krll; subR[32] = krlr;
        /* kw4 */
        subL[33] = krrl; subR[33] = krrr;

        camellia_setup_tail(subkey, subL, subR, 32);
}

static void camellia_setup192(const unsigned char *key, u32 *subkey)
{
        unsigned char kk[32];
        u32 krll, krlr, krrl, krrr;

        memcpy(kk, key, 24);
        memcpy((unsigned char *)&krll, key+16, 4);
        memcpy((unsigned char *)&krlr, key+20, 4);
        krrl = ~krll;
        krrr = ~krlr;
        memcpy(kk+24, (unsigned char *)&krrl, 4);
        memcpy(kk+28, (unsigned char *)&krrr, 4);
        camellia_setup256(kk, subkey);
}


/*
 * Encrypt/decrypt
 */
#define CAMELLIA_FLS(ll, lr, rl, rr, kll, klr, krl, krr, t0, t1, t2, t3) ({ \
        t0 = kll;                                                        \
        t2 = krr;                                                        \
        t0 &= ll;                                                        \
        t2 |= rr;                                                        \
        rl ^= t2;                                                        \
        lr ^= rol32(t0, 1);                                                \
        t3 = krl;                                                        \
        t1 = klr;                                                        \
        t3 &= rl;                                                        \
        t1 |= lr;                                                        \
        ll ^= t1;                                                        \
        rr ^= rol32(t3, 1);                                                \
})

#define CAMELLIA_ROUNDSM(xl, xr, kl, kr, yl, yr, il, ir) ({                \
        yl ^= kl;                                                        \
        yr ^= kr;                                                        \
        ir =  camellia_sp1110[(u8)xr];                                        \
        il =  camellia_sp1110[(u8)(xl >> 24)];                                \
        ir ^= camellia_sp0222[(u8)(xr >> 24)];                                \
        il ^= camellia_sp0222[(u8)(xl >> 16)];                                \
        ir ^= camellia_sp3033[(u8)(xr >> 16)];                                \
        il ^= camellia_sp3033[(u8)(xl >> 8)];                                \
        ir ^= camellia_sp4404[(u8)(xr >> 8)];                                \
        il ^= camellia_sp4404[(u8)xl];                                        \
        ir ^= il;                                                        \
        yl ^= ir;                                                        \
        yr ^= ror32(il, 8) ^ ir;                                        \
})

/* max = 24: 128bit encrypt, max = 32: 256bit encrypt */
static void camellia_do_encrypt(const u32 *subkey, u32 *io, unsigned max)
{
        u32 il, ir, t0, t1;            /* temporary variables */

        /* pre whitening but absorb kw2 */
        io[0] ^= SUBKEY_L(0);
        io[1] ^= SUBKEY_R(0);

        /* main iteration */
#define ROUNDS(i) ({ \
        CAMELLIA_ROUNDSM(io[0], io[1], \
                         SUBKEY_L(i + 2), SUBKEY_R(i + 2), \
                         io[2], io[3], il, ir); \
        CAMELLIA_ROUNDSM(io[2], io[3], \
                         SUBKEY_L(i + 3), SUBKEY_R(i + 3), \
                         io[0], io[1], il, ir); \
        CAMELLIA_ROUNDSM(io[0], io[1], \
                         SUBKEY_L(i + 4), SUBKEY_R(i + 4), \
                         io[2], io[3], il, ir); \
        CAMELLIA_ROUNDSM(io[2], io[3], \
                         SUBKEY_L(i + 5), SUBKEY_R(i + 5), \
                         io[0], io[1], il, ir); \
        CAMELLIA_ROUNDSM(io[0], io[1], \
                         SUBKEY_L(i + 6), SUBKEY_R(i + 6), \
                         io[2], io[3], il, ir); \
        CAMELLIA_ROUNDSM(io[2], io[3], \
                         SUBKEY_L(i + 7), SUBKEY_R(i + 7), \
                         io[0], io[1], il, ir); \
})
#define FLS(i) ({ \
        CAMELLIA_FLS(io[0], io[1], io[2], io[3], \
                     SUBKEY_L(i + 0), SUBKEY_R(i + 0), \
                     SUBKEY_L(i + 1), SUBKEY_R(i + 1), \
                     t0, t1, il, ir); \
})

        ROUNDS(0);
        FLS(8);
        ROUNDS(8);
        FLS(16);
        ROUNDS(16);
        if (max == 32) {
                FLS(24);
                ROUNDS(24);
        }

#undef ROUNDS
#undef FLS

        /* post whitening but kw4 */
        io[2] ^= SUBKEY_L(max);
        io[3] ^= SUBKEY_R(max);
        /* NB: io[0],[1] should be swapped with [2],[3] by caller! */
}

static void camellia_do_decrypt(const u32 *subkey, u32 *io, unsigned i)
{
        u32 il, ir, t0, t1;            /* temporary variables */

        /* pre whitening but absorb kw2 */
        io[0] ^= SUBKEY_L(i);
        io[1] ^= SUBKEY_R(i);

        /* main iteration */
#define ROUNDS(i) ({ \
        CAMELLIA_ROUNDSM(io[0], io[1], \
                         SUBKEY_L(i + 7), SUBKEY_R(i + 7), \
                         io[2], io[3], il, ir); \
        CAMELLIA_ROUNDSM(io[2], io[3], \
                         SUBKEY_L(i + 6), SUBKEY_R(i + 6), \
                         io[0], io[1], il, ir); \
        CAMELLIA_ROUNDSM(io[0], io[1], \
                         SUBKEY_L(i + 5), SUBKEY_R(i + 5), \
                         io[2], io[3], il, ir); \
        CAMELLIA_ROUNDSM(io[2], io[3], \
                         SUBKEY_L(i + 4), SUBKEY_R(i + 4), \
                         io[0], io[1], il, ir); \
        CAMELLIA_ROUNDSM(io[0], io[1], \
                         SUBKEY_L(i + 3), SUBKEY_R(i + 3), \
                         io[2], io[3], il, ir); \
        CAMELLIA_ROUNDSM(io[2], io[3], \
                         SUBKEY_L(i + 2), SUBKEY_R(i + 2), \
                         io[0], io[1], il, ir); \
})
#define FLS(i) ({ \
        CAMELLIA_FLS(io[0], io[1], io[2], io[3], \
                     SUBKEY_L(i + 1), SUBKEY_R(i + 1), \
                     SUBKEY_L(i + 0), SUBKEY_R(i + 0), \
                     t0, t1, il, ir); \
})

        if (i == 32) {
                ROUNDS(24);
                FLS(24);
        }
        ROUNDS(16);
        FLS(16);
        ROUNDS(8);
        FLS(8);
        ROUNDS(0);

#undef ROUNDS
#undef FLS

        /* post whitening but kw4 */
        io[2] ^= SUBKEY_L(0);
        io[3] ^= SUBKEY_R(0);
        /* NB: 0,1 should be swapped with 2,3 by caller! */
}


struct camellia_ctx {
        int key_length;
        u32 key_table[CAMELLIA_TABLE_BYTE_LEN / sizeof(u32)];
};

static int
camellia_set_key(struct crypto_tfm *tfm, const u8 *in_key,
                 unsigned int key_len)
{
        struct camellia_ctx *cctx = crypto_tfm_ctx(tfm);
        const unsigned char *key = (const unsigned char *)in_key;

        if (key_len != 16 && key_len != 24 && key_len != 32)
                return -EINVAL;

        cctx->key_length = key_len;

        switch (key_len) {
        case 16:
                camellia_setup128(key, cctx->key_table);
                break;
        case 24:
                camellia_setup192(key, cctx->key_table);
                break;
        case 32:
                camellia_setup256(key, cctx->key_table);
                break;
        }

        return 0;
}

static void camellia_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
        const struct camellia_ctx *cctx = crypto_tfm_ctx(tfm);
        unsigned int max;

        u32 tmp[4];

        tmp[0] = get_unaligned_be32(in);
        tmp[1] = get_unaligned_be32(in + 4);
        tmp[2] = get_unaligned_be32(in + 8);
        tmp[3] = get_unaligned_be32(in + 12);

        if (cctx->key_length == 16)
                max = 24;
        else
                max = 32; /* for key lengths of 24 and 32 */

        camellia_do_encrypt(cctx->key_table, tmp, max);

        /* do_encrypt returns 0,1 swapped with 2,3 */
        put_unaligned_be32(tmp[2], out);
        put_unaligned_be32(tmp[3], out + 4);
        put_unaligned_be32(tmp[0], out + 8);
        put_unaligned_be32(tmp[1], out + 12);
}

static void camellia_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
        const struct camellia_ctx *cctx = crypto_tfm_ctx(tfm);
        unsigned int max;

        u32 tmp[4];

        tmp[0] = get_unaligned_be32(in);
        tmp[1] = get_unaligned_be32(in + 4);
        tmp[2] = get_unaligned_be32(in + 8);
        tmp[3] = get_unaligned_be32(in + 12);

        if (cctx->key_length == 16)
                max = 24;
        else
                max = 32; /* for key lengths of 24 and 32 */

        camellia_do_decrypt(cctx->key_table, tmp, max);

        /* do_decrypt returns 0,1 swapped with 2,3 */
        put_unaligned_be32(tmp[2], out);
        put_unaligned_be32(tmp[3], out + 4);
        put_unaligned_be32(tmp[0], out + 8);
        put_unaligned_be32(tmp[1], out + 12);
}

static struct crypto_alg camellia_alg = {
        .cra_name                =        "camellia",
        .cra_driver_name        =        "camellia-generic",
        .cra_priority                =        100,
        .cra_flags                =        CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                =        CAMELLIA_BLOCK_SIZE,
        .cra_ctxsize                =        sizeof(struct camellia_ctx),
        .cra_module                =        THIS_MODULE,
        .cra_u                        =        {
                .cipher = {
                        .cia_min_keysize        =        CAMELLIA_MIN_KEY_SIZE,
                        .cia_max_keysize        =        CAMELLIA_MAX_KEY_SIZE,
                        .cia_setkey                =        camellia_set_key,
                        .cia_encrypt                =        camellia_encrypt,
                        .cia_decrypt                =        camellia_decrypt
                }
        }
};

static int __init camellia_init(void)
{
        return crypto_register_alg(&camellia_alg);
}

static void __exit camellia_fini(void)
{
        crypto_unregister_alg(&camellia_alg);
}

subsys_initcall(camellia_init);
module_exit(camellia_fini);

MODULE_DESCRIPTION("Camellia Cipher Algorithm");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("camellia");
MODULE_ALIAS_CRYPTO("camellia-generic");









































   27 

   25 











   27 

    3 











   28 

    4 











   28 
    8 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Crypto library utility functions
 *
 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 */

#include <linux/unaligned.h>
#include <crypto/utils.h>
#include <linux/module.h>

/*
 * XOR @len bytes from @src1 and @src2 together, writing the result to @dst
 * (which may alias one of the sources).  Don't call this directly; call
 * crypto_xor() or crypto_xor_cpy() instead.
 */
void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len)
{
        int relalign = 0;

        if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
                int size = sizeof(unsigned long);
                int d = (((unsigned long)dst ^ (unsigned long)src1) |
                         ((unsigned long)dst ^ (unsigned long)src2)) &
                        (size - 1);

                relalign = d ? 1 << __ffs(d) : size;

                /*
                 * If we care about alignment, process as many bytes as
                 * needed to advance dst and src to values whose alignments
                 * equal their relative alignment. This will allow us to
                 * process the remainder of the input using optimal strides.
                 */
                while (((unsigned long)dst & (relalign - 1)) && len > 0) {
                        *dst++ = *src1++ ^ *src2++;
                        len--;
                }
        }

        while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) {
                if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
                        u64 l = get_unaligned((u64 *)src1) ^
                                get_unaligned((u64 *)src2);
                        put_unaligned(l, (u64 *)dst);
                } else {
                        *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2;
                }
                dst += 8;
                src1 += 8;
                src2 += 8;
                len -= 8;
        }

        while (len >= 4 && !(relalign & 3)) {
                if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
                        u32 l = get_unaligned((u32 *)src1) ^
                                get_unaligned((u32 *)src2);
                        put_unaligned(l, (u32 *)dst);
                } else {
                        *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2;
                }
                dst += 4;
                src1 += 4;
                src2 += 4;
                len -= 4;
        }

        while (len >= 2 && !(relalign & 1)) {
                if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
                        u16 l = get_unaligned((u16 *)src1) ^
                                get_unaligned((u16 *)src2);
                        put_unaligned(l, (u16 *)dst);
                } else {
                        *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2;
                }
                dst += 2;
                src1 += 2;
                src2 += 2;
                len -= 2;
        }

        while (len--)
                *dst++ = *src1++ ^ *src2++;
}
EXPORT_SYMBOL_GPL(__crypto_xor);

MODULE_DESCRIPTION("Crypto library utility functions");
MODULE_LICENSE("GPL");









































































   99 









    1 









    1 





   98 










    1 


  100 




































    5 



    7 


    7 

    7 


















































   69 






































   70 

   70 
   70 






   70 
    1 




   67 
    1 



   69 
    2 

   70 







   70 

    6 


    6 






    6 
    6 




    6 




   63 
    8 













   70 




   70 
















   70 




   70 
    8 





   70 
   70 

   70 
    2 






   70 

    6 

    6 

    6 
    6 



   69 







   22 



























   22 







   22 




   22 

   22 
   22 

   22 





   22 
    1 



   22 
    1 

   22 




   22 
    2 


    2 





    2 
    2 




    2 



   20 
    3 


   22 












   22 


   22 


   22 



   22 
   22 
    3 




   22 
   22 

   21 
    1 





   22 
    2 


    2 

    2 
    2 




   22 


   22 

   22 












































   23 








   23 











   23 


   23 

























   22 










   22 


   22 

   22 






















































































   61 










    1 

   63 



   62 


   63 








   25 














   25 















































   25 






    5 



   25 












   25 
   24 

   25 
   24 


   25 










    2 








    2 




    2 







    2 












    2 

    2 












    3 

    3 













    3 

    3 















   15 
    7 
    7 





   15 












    9 

   15 















































    6 




    6 

    2 



    2 




    5 


    6 
    1 






    1 











   10 





   10 


















    2 






    2 








    2 





































































   10 








    9 















   10 









    7 




    7 











    7 


    7 











































































































    1 









    1 




    1 












    1 























  100 



  102 
    7 

  101 













  100 






    8 







   35 
   27 


    8 
















  102 




   99 




  102 




   93 
  102 





  100 




  102 
























   85 

  100 

   86 






  108 












  107 
   51 

  107 





   66 





  108 
  108 







   94 























   63 


   62 




   63 


   63 












   30 



   29 







   29 


   29 
   29 

   29 


   29 









   30 




   29 














   25 



   25 





   25 






























   22 
   22 








   22 


   22 


















   22 






































   22 
























   22 



   22 

































   22 




   22 
   22 












   22 


   22 

   22 























   22 







   22 











   22 
   17 




   22 














   22 

















































































































    6 




    6 
    6 















    6 


    6 








    3 


    3 
    3 

    1 
    1 









    2 
    2 
















































































































   27 












    6 






















   26 




    4 







    4 








    4 







    4 







    4 



    4 







    4 





    4 






























   27 
   27 
   27 
   26 
   27 

   27 
   27 










   27 





   27 




   27 
   27 




























   25 

















   26 



   26 

   26 









   26 










   23 
    2 








   23 
   23 








   23 

   23 
    3 







   23 
















    1 



   22 

   22 




   23 








   23 

   23 



   23 












   22 



   23 

   23 












   23 











   23 



















   26 
















    5 





    9 


    9 


    9 

    9 



















   22 






   15 
    7 
    7 


   22 



   21 
   22 

   22 



   17 
   17 










   23 


   22 












   23 
   23 







    1 



























    3 



   23 
   23 






    2 



    2 


    2 






    2 



    2 








    2 



    2 


    2 


    2 























   77 


   74 


   77 





   77 

























































































































































































































































































































































































































































































































































































































































































































































































































    1 











    1 



    1 
    1 

























































    4 












































    3 








    3 




    3 








    2 


    3 
    2 






    2 

















































    1 



    1 













    1 
    1 

























    4 



    4 










    4 




















































    4 
    4 


    3 

    4 
    4 

    3 





    2 
    2 



    4 
    4 










    1 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
// SPDX-License-Identifier: GPL-2.0-or-later
/* SCTP kernel implementation
 * (C) Copyright IBM Corp. 2001, 2004
 * Copyright (c) 1999-2000 Cisco, Inc.
 * Copyright (c) 1999-2001 Motorola, Inc.
 * Copyright (c) 2001-2002 Intel Corp.
 *
 * This file is part of the SCTP kernel implementation
 *
 * These functions work with the state functions in sctp_sm_statefuns.c
 * to implement the state operations.  These functions implement the
 * steps which require modifying existing data structures.
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *    La Monte H.P. Yarroll <piggy@acm.org>
 *    Karl Knutson          <karl@athena.chicago.il.us>
 *    C. Robin              <chris@hundredacre.ac.uk>
 *    Jon Grimm             <jgrimm@us.ibm.com>
 *    Xingang Guo           <xingang.guo@intel.com>
 *    Dajiang Zhang            <dajiang.zhang@nokia.com>
 *    Sridhar Samudrala            <sri@us.ibm.com>
 *    Daisy Chang            <daisyc@us.ibm.com>
 *    Ardelle Fan            <ardelle.fan@intel.com>
 *    Kevin Gao             <kevin.gao@intel.com>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <crypto/hash.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/net.h>
#include <linux/inet.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <net/sock.h>

#include <linux/skbuff.h>
#include <linux/random.h>        /* for get_random_bytes */
#include <net/sctp/sctp.h>
#include <net/sctp/sm.h>

static struct sctp_chunk *sctp_make_control(const struct sctp_association *asoc,
                                            __u8 type, __u8 flags, int paylen,
                                            gfp_t gfp);
static struct sctp_chunk *sctp_make_data(const struct sctp_association *asoc,
                                         __u8 flags, int paylen, gfp_t gfp);
static struct sctp_chunk *_sctp_make_chunk(const struct sctp_association *asoc,
                                           __u8 type, __u8 flags, int paylen,
                                           gfp_t gfp);
static struct sctp_cookie_param *sctp_pack_cookie(
                                        const struct sctp_endpoint *ep,
                                        const struct sctp_association *asoc,
                                        const struct sctp_chunk *init_chunk,
                                        int *cookie_len,
                                        const __u8 *raw_addrs, int addrs_len);
static int sctp_process_param(struct sctp_association *asoc,
                              union sctp_params param,
                              const union sctp_addr *peer_addr,
                              gfp_t gfp);
static void *sctp_addto_param(struct sctp_chunk *chunk, int len,
                              const void *data);

/* Control chunk destructor */
static void sctp_control_release_owner(struct sk_buff *skb)
{
        struct sctp_chunk *chunk = skb_shinfo(skb)->destructor_arg;

        if (chunk->shkey) {
                struct sctp_shared_key *shkey = chunk->shkey;
                struct sctp_association *asoc = chunk->asoc;

                /* refcnt == 2 and !list_empty mean after this release, it's
                 * not being used anywhere, and it's time to notify userland
                 * that this shkey can be freed if it's been deactivated.
                 */
                if (shkey->deactivated && !list_empty(&shkey->key_list) &&
                    refcount_read(&shkey->refcnt) == 2) {
                        struct sctp_ulpevent *ev;

                        ev = sctp_ulpevent_make_authkey(asoc, shkey->key_id,
                                                        SCTP_AUTH_FREE_KEY,
                                                        GFP_KERNEL);
                        if (ev)
                                asoc->stream.si->enqueue_event(&asoc->ulpq, ev);
                }
                sctp_auth_shkey_release(chunk->shkey);
        }
}

static void sctp_control_set_owner_w(struct sctp_chunk *chunk)
{
        struct sctp_association *asoc = chunk->asoc;
        struct sk_buff *skb = chunk->skb;

        /* TODO: properly account for control chunks.
         * To do it right we'll need:
         *  1) endpoint if association isn't known.
         *  2) proper memory accounting.
         *
         *  For now don't do anything for now.
         */
        if (chunk->auth) {
                chunk->shkey = asoc->shkey;
                sctp_auth_shkey_hold(chunk->shkey);
        }
        skb->sk = asoc ? asoc->base.sk : NULL;
        skb_shinfo(skb)->destructor_arg = chunk;
        skb->destructor = sctp_control_release_owner;
}

/* What was the inbound interface for this chunk? */
int sctp_chunk_iif(const struct sctp_chunk *chunk)
{
        struct sk_buff *skb = chunk->skb;

        return SCTP_INPUT_CB(skb)->af->skb_iif(skb);
}

/* RFC 2960 3.3.2 Initiation (INIT) (1)
 *
 * Note 2: The ECN capable field is reserved for future use of
 * Explicit Congestion Notification.
 */
static const struct sctp_paramhdr ecap_param = {
        SCTP_PARAM_ECN_CAPABLE,
        cpu_to_be16(sizeof(struct sctp_paramhdr)),
};
static const struct sctp_paramhdr prsctp_param = {
        SCTP_PARAM_FWD_TSN_SUPPORT,
        cpu_to_be16(sizeof(struct sctp_paramhdr)),
};

/* A helper to initialize an op error inside a provided chunk, as most
 * cause codes will be embedded inside an abort chunk.
 */
int sctp_init_cause(struct sctp_chunk *chunk, __be16 cause_code,
                    size_t paylen)
{
        struct sctp_errhdr err;
        __u16 len;

        /* Cause code constants are now defined in network order.  */
        err.cause = cause_code;
        len = sizeof(err) + paylen;
        err.length = htons(len);

        if (skb_tailroom(chunk->skb) < len)
                return -ENOSPC;

        chunk->subh.err_hdr = sctp_addto_chunk(chunk, sizeof(err), &err);

        return 0;
}

/* 3.3.2 Initiation (INIT) (1)
 *
 * This chunk is used to initiate a SCTP association between two
 * endpoints. The format of the INIT chunk is shown below:
 *
 *     0                   1                   2                   3
 *     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *    |   Type = 1    |  Chunk Flags  |      Chunk Length             |
 *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *    |                         Initiate Tag                          |
 *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *    |           Advertised Receiver Window Credit (a_rwnd)          |
 *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *    |  Number of Outbound Streams   |  Number of Inbound Streams    |
 *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *    |                          Initial TSN                          |
 *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *    \                                                               \
 *    /              Optional/Variable-Length Parameters              /
 *    \                                                               \
 *    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *
 * The INIT chunk contains the following parameters. Unless otherwise
 * noted, each parameter MUST only be included once in the INIT chunk.
 *
 * Fixed Parameters                     Status
 * ----------------------------------------------
 * Initiate Tag                        Mandatory
 * Advertised Receiver Window Credit   Mandatory
 * Number of Outbound Streams          Mandatory
 * Number of Inbound Streams           Mandatory
 * Initial TSN                         Mandatory
 *
 * Variable Parameters                  Status     Type Value
 * -------------------------------------------------------------
 * IPv4 Address (Note 1)               Optional    5
 * IPv6 Address (Note 1)               Optional    6
 * Cookie Preservative                 Optional    9
 * Reserved for ECN Capable (Note 2)   Optional    32768 (0x8000)
 * Host Name Address (Note 3)          Optional    11
 * Supported Address Types (Note 4)    Optional    12
 */
struct sctp_chunk *sctp_make_init(const struct sctp_association *asoc,
                                  const struct sctp_bind_addr *bp,
                                  gfp_t gfp, int vparam_len)
{
        struct sctp_supported_ext_param ext_param;
        struct sctp_adaptation_ind_param aiparam;
        struct sctp_paramhdr *auth_chunks = NULL;
        struct sctp_paramhdr *auth_hmacs = NULL;
        struct sctp_supported_addrs_param sat;
        struct sctp_endpoint *ep = asoc->ep;
        struct sctp_chunk *retval = NULL;
        int num_types, addrs_len = 0;
        struct sctp_inithdr init;
        union sctp_params addrs;
        struct sctp_sock *sp;
        __u8 extensions[5];
        size_t chunksize;
        __be16 types[2];
        int num_ext = 0;

        /* RFC 2960 3.3.2 Initiation (INIT) (1)
         *
         * Note 1: The INIT chunks can contain multiple addresses that
         * can be IPv4 and/or IPv6 in any combination.
         */

        /* Convert the provided bind address list to raw format. */
        addrs = sctp_bind_addrs_to_raw(bp, &addrs_len, gfp);

        init.init_tag                   = htonl(asoc->c.my_vtag);
        init.a_rwnd                   = htonl(asoc->rwnd);
        init.num_outbound_streams  = htons(asoc->c.sinit_num_ostreams);
        init.num_inbound_streams   = htons(asoc->c.sinit_max_instreams);
        init.initial_tsn           = htonl(asoc->c.initial_tsn);

        /* How many address types are needed? */
        sp = sctp_sk(asoc->base.sk);
        num_types = sp->pf->supported_addrs(sp, types);

        chunksize = sizeof(init) + addrs_len;
        chunksize += SCTP_PAD4(SCTP_SAT_LEN(num_types));

        if (asoc->ep->ecn_enable)
                chunksize += sizeof(ecap_param);

        if (asoc->ep->prsctp_enable)
                chunksize += sizeof(prsctp_param);

        /* ADDIP: Section 4.2.7:
         *  An implementation supporting this extension [ADDIP] MUST list
         *  the ASCONF,the ASCONF-ACK, and the AUTH  chunks in